src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(tarball_suffix, "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Darwin': 'apple-darwin',
 181         'DragonFly': 'unknown-dragonfly',
 182         'FreeBSD': 'unknown-freebsd',
 183         'Haiku': 'unknown-haiku',
 184         'NetBSD': 'unknown-netbsd',
 185         'OpenBSD': 'unknown-openbsd'
 186     }
 187
 188     # Consider the direct transformation first and then the special cases
 189     if ostype in ostype_mapper:
 190         ostype = ostype_mapper[ostype]
 191     elif ostype == 'Linux':
 192         os_from_sp = subprocess.check_output(
 193             ['uname', '-o']).strip().decode(default_encoding)
 194         if os_from_sp == 'Android':
 195             ostype = 'linux-android'
 196         else:
 197             ostype = 'unknown-linux-gnu'
 198     elif ostype == 'SunOS':
 199         ostype = 'sun-solaris'
 200         # On Solaris, uname -m will return a machine classification instead
 201         # of a cpu type, so uname -p is recommended instead.  However, the
 202         # output from that option is too generic for our purposes (it will
 203         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 204         # must be used instead.
 205         try:
 206             cputype = subprocess.check_output(
 207                 ['isainfo', '-k']).strip().decode(default_encoding)
 208         except (subprocess.CalledProcessError, OSError):
 209             err = "isainfo not found"
 210             sys.exit(err)
 211     elif ostype.startswith('MINGW'):
 212         # msys' `uname` does not print gcc configuration, but prints msys
 213         # configuration. so we cannot believe `uname -m`:
 214         # msys1 is always i686 and msys2 is always x86_64.
 215         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 216         # MINGW64 on x86_64.
 217         ostype = 'pc-windows-gnu'
 218         cputype = 'i686'
 219         if os.environ.get('MSYSTEM') == 'MINGW64':
 220             cputype = 'x86_64'
 221     elif ostype.startswith('MSYS'):
 222         ostype = 'pc-windows-gnu'
 223     elif ostype.startswith('CYGWIN_NT'):
 224         cputype = 'i686'
 225         if ostype.endswith('WOW64'):
 226             cputype = 'x86_64'
 227         ostype = 'pc-windows-gnu'
 228     else:
 229         err = "unknown OS type: {}".format(ostype)
 230         sys.exit(err)
 231
 232     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 233         cputype = subprocess.check_output(
 234               ['uname', '-p']).strip().decode(default_encoding)
 235     cputype_mapper = {
 236         'BePC': 'i686',
 237         'aarch64': 'aarch64',
 238         'amd64': 'x86_64',
 239         'arm64': 'aarch64',
 240         'i386': 'i686',
 241         'i486': 'i686',
 242         'i686': 'i686',
 243         'i786': 'i686',
 244         'powerpc': 'powerpc',
 245         'powerpc64': 'powerpc64',
 246         'powerpc64le': 'powerpc64le',
 247         'ppc': 'powerpc',
 248         'ppc64': 'powerpc64',
 249         'ppc64le': 'powerpc64le',
 250         's390x': 's390x',
 251         'x64': 'x86_64',
 252         'x86': 'i686',
 253         'x86-64': 'x86_64',
 254         'x86_64': 'x86_64'
 255     }
 256
 257     # Consider the direct transformation first and then the special cases
 258     if cputype in cputype_mapper:
 259         cputype = cputype_mapper[cputype]
 260     elif cputype in {'xscale', 'arm'}:
 261         cputype = 'arm'
 262         if ostype == 'linux-android':
 263             ostype = 'linux-androideabi'
 264         elif ostype == 'unknown-freebsd':
 265             cputype = subprocess.check_output(
 266                 ['uname', '-p']).strip().decode(default_encoding)
 267             ostype = 'unknown-freebsd'
 268     elif cputype == 'armv6l':
 269         cputype = 'arm'
 270         if ostype == 'linux-android':
 271             ostype = 'linux-androideabi'
 272         else:
 273             ostype += 'eabihf'
 274     elif cputype in {'armv7l', 'armv8l'}:
 275         cputype = 'armv7'
 276         if ostype == 'linux-android':
 277             ostype = 'linux-androideabi'
 278         else:
 279             ostype += 'eabihf'
 280     elif cputype == 'mips':
 281         if sys.byteorder == 'big':
 282             cputype = 'mips'
 283         elif sys.byteorder == 'little':
 284             cputype = 'mipsel'
 285         else:
 286             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 287     elif cputype == 'mips64':
 288         if sys.byteorder == 'big':
 289             cputype = 'mips64'
 290         elif sys.byteorder == 'little':
 291             cputype = 'mips64el'
 292         else:
 293             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 294         # only the n64 ABI is supported, indicate it
 295         ostype += 'abi64'
 296     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 297         pass
 298     else:
 299         err = "unknown cpu type: {}".format(cputype)
 300         sys.exit(err)
 301
 302     return "{}-{}".format(cputype, ostype)
 303
 304
 305 @contextlib.contextmanager
 306 def output(filepath):
 307     tmp = filepath + '.tmp'
 308     with open(tmp, 'w') as f:
 309         yield f
 310     try:
 311         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 312         os.rename(tmp, filepath)
 313     except OSError:
 314         shutil.copy2(tmp, filepath)
 315         os.remove(tmp)
 316
 317
 318 class RustBuild(object):
 319     """Provide all the methods required to build Rust"""
 320     def __init__(self):
 321         self.cargo_channel = ''
 322         self.date = ''
 323         self._download_url = ''
 324         self.rustc_channel = ''
 325         self.rustfmt_channel = ''
 326         self.build = ''
 327         self.build_dir = os.path.join(os.getcwd(), "build")
 328         self.clean = False
 329         self.config_toml = ''
 330         self.rust_root = ''
 331         self.use_locked_deps = ''
 332         self.use_vendored_sources = ''
 333         self.verbose = False
 334
 335
 336     def download_stage0(self):
 337         """Fetch the build system for Rust, written in Rust
 338
 339         This method will build a cache directory, then it will fetch the
 340         tarball which has the stage0 compiler used to then bootstrap the Rust
 341         compiler itself.
 342
 343         Each downloaded tarball is extracted, after that, the script
 344         will move all the content to the right place.
 345         """
 346         rustc_channel = self.rustc_channel
 347         cargo_channel = self.cargo_channel
 348         rustfmt_channel = self.rustfmt_channel
 349
 350         def support_xz():
 351             try:
 352                 with tempfile.NamedTemporaryFile(delete=False) as temp_file:
 353                     temp_path = temp_file.name
 354                 with tarfile.open(temp_path, "w:xz") as tar:
 355                     pass
 356                 return True
 357             except tarfile.CompressionError:
 358                 return False
 359
 360         if self.rustc().startswith(self.bin_root()) and \
 361                 (not os.path.exists(self.rustc()) or
 362                  self.program_out_of_date(self.rustc_stamp())):
 363             if os.path.exists(self.bin_root()):
 364                 shutil.rmtree(self.bin_root())
 365             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 366             filename = "rust-std-{}-{}{}".format(
 367                 rustc_channel, self.build, tarball_suffix)
 368             pattern = "rust-std-{}".format(self.build)
 369             self._download_stage0_helper(filename, pattern, tarball_suffix)
 370
 371             filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
 372                                               tarball_suffix)
 373             self._download_stage0_helper(filename, "rustc", tarball_suffix)
 374             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 375             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 376             with output(self.rustc_stamp()) as rust_stamp:
 377                 rust_stamp.write(self.date)
 378
 379             # This is required so that we don't mix incompatible MinGW
 380             # libraries/binaries that are included in rust-std with
 381             # the system MinGW ones.
 382             if "pc-windows-gnu" in self.build:
 383                 filename = "rust-mingw-{}-{}{}".format(
 384                     rustc_channel, self.build, tarball_suffix)
 385                 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
 386
 387         if self.cargo().startswith(self.bin_root()) and \
 388                 (not os.path.exists(self.cargo()) or
 389                  self.program_out_of_date(self.cargo_stamp())):
 390             tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 391             filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
 392                                               tarball_suffix)
 393             self._download_stage0_helper(filename, "cargo", tarball_suffix)
 394             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 395             with output(self.cargo_stamp()) as cargo_stamp:
 396                 cargo_stamp.write(self.date)
 397
 398         if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
 399             not os.path.exists(self.rustfmt())
 400             or self.program_out_of_date(self.rustfmt_stamp())
 401         ):
 402             if rustfmt_channel:
 403                 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
 404                 [channel, date] = rustfmt_channel.split('-', 1)
 405                 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
 406                 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
 407                 self.fix_executable("{}/bin/rustfmt".format(self.bin_root()))
 408                 self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root()))
 409                 with output(self.rustfmt_stamp()) as rustfmt_stamp:
 410                     rustfmt_stamp.write(self.date)
 411
 412     def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
 413         if date is None:
 414             date = self.date
 415         cache_dst = os.path.join(self.build_dir, "cache")
 416         rustc_cache = os.path.join(cache_dst, date)
 417         if not os.path.exists(rustc_cache):
 418             os.makedirs(rustc_cache)
 419
 420         url = "{}/dist/{}".format(self._download_url, date)
 421         tarball = os.path.join(rustc_cache, filename)
 422         if not os.path.exists(tarball):
 423             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 424         unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
 425
 426     @staticmethod
 427     def fix_executable(fname):
 428         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 429
 430         This method is only required on NixOS and uses the PatchELF utility to
 431         change the dynamic linker of ELF executables.
 432
 433         Please see https://nixos.org/patchelf.html for more information
 434         """
 435         default_encoding = sys.getdefaultencoding()
 436         try:
 437             ostype = subprocess.check_output(
 438                 ['uname', '-s']).strip().decode(default_encoding)
 439         except subprocess.CalledProcessError:
 440             return
 441         except OSError as reason:
 442             if getattr(reason, 'winerror', None) is not None:
 443                 return
 444             raise reason
 445
 446         if ostype != "Linux":
 447             return
 448
 449         if not os.path.exists("/etc/NIXOS"):
 450             return
 451         if os.path.exists("/lib"):
 452             return
 453
 454         # At this point we're pretty sure the user is running NixOS
 455         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 456         print(nix_os_msg, fname)
 457
 458         try:
 459             interpreter = subprocess.check_output(
 460                 ["patchelf", "--print-interpreter", fname])
 461             interpreter = interpreter.strip().decode(default_encoding)
 462         except subprocess.CalledProcessError as reason:
 463             print("warning: failed to call patchelf:", reason)
 464             return
 465
 466         loader = interpreter.split("/")[-1]
 467
 468         try:
 469             ldd_output = subprocess.check_output(
 470                 ['ldd', '/run/current-system/sw/bin/sh'])
 471             ldd_output = ldd_output.strip().decode(default_encoding)
 472         except subprocess.CalledProcessError as reason:
 473             print("warning: unable to call ldd:", reason)
 474             return
 475
 476         for line in ldd_output.splitlines():
 477             libname = line.split()[0]
 478             if libname.endswith(loader):
 479                 loader_path = libname[:len(libname) - len(loader)]
 480                 break
 481         else:
 482             print("warning: unable to find the path to the dynamic linker")
 483             return
 484
 485         correct_interpreter = loader_path + loader
 486
 487         try:
 488             subprocess.check_output(
 489                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 490         except subprocess.CalledProcessError as reason:
 491             print("warning: failed to call patchelf:", reason)
 492             return
 493
 494     def rustc_stamp(self):
 495         """Return the path for .rustc-stamp
 496
 497         >>> rb = RustBuild()
 498         >>> rb.build_dir = "build"
 499         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 500         True
 501         """
 502         return os.path.join(self.bin_root(), '.rustc-stamp')
 503
 504     def cargo_stamp(self):
 505         """Return the path for .cargo-stamp
 506
 507         >>> rb = RustBuild()
 508         >>> rb.build_dir = "build"
 509         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 510         True
 511         """
 512         return os.path.join(self.bin_root(), '.cargo-stamp')
 513
 514     def rustfmt_stamp(self):
 515         """Return the path for .rustfmt-stamp
 516
 517         >>> rb = RustBuild()
 518         >>> rb.build_dir = "build"
 519         >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
 520         True
 521         """
 522         return os.path.join(self.bin_root(), '.rustfmt-stamp')
 523
 524     def program_out_of_date(self, stamp_path):
 525         """Check if the given program stamp is out of date"""
 526         if not os.path.exists(stamp_path) or self.clean:
 527             return True
 528         with open(stamp_path, 'r') as stamp:
 529             return self.date != stamp.read()
 530
 531     def bin_root(self):
 532         """Return the binary root directory
 533
 534         >>> rb = RustBuild()
 535         >>> rb.build_dir = "build"
 536         >>> rb.bin_root() == os.path.join("build", "stage0")
 537         True
 538
 539         When the 'build' property is given should be a nested directory:
 540
 541         >>> rb.build = "devel"
 542         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 543         True
 544         """
 545         return os.path.join(self.build_dir, self.build, "stage0")
 546
 547     def get_toml(self, key, section=None):
 548         """Returns the value of the given key in config.toml, otherwise returns None
 549
 550         >>> rb = RustBuild()
 551         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 552         >>> rb.get_toml("key2")
 553         'value2'
 554
 555         If the key does not exists, the result is None:
 556
 557         >>> rb.get_toml("key3") is None
 558         True
 559
 560         Optionally also matches the section the key appears in
 561
 562         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 563         >>> rb.get_toml('key', 'a')
 564         'value1'
 565         >>> rb.get_toml('key', 'b')
 566         'value2'
 567         >>> rb.get_toml('key', 'c') is None
 568         True
 569
 570         >>> rb.config_toml = 'key1 = true'
 571         >>> rb.get_toml("key1")
 572         'true'
 573         """
 574
 575         cur_section = None
 576         for line in self.config_toml.splitlines():
 577             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 578             if section_match is not None:
 579                 cur_section = section_match.group(1)
 580
 581             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 582             if match is not None:
 583                 value = match.group(1)
 584                 if section is None or section == cur_section:
 585                     return self.get_string(value) or value.strip()
 586         return None
 587
 588     def cargo(self):
 589         """Return config path for cargo"""
 590         return self.program_config('cargo')
 591
 592     def rustc(self):
 593         """Return config path for rustc"""
 594         return self.program_config('rustc')
 595
 596     def rustfmt(self):
 597         """Return config path for rustfmt"""
 598         if not self.rustfmt_channel:
 599             return None
 600         return self.program_config('rustfmt')
 601
 602     def program_config(self, program):
 603         """Return config path for the given program
 604
 605         >>> rb = RustBuild()
 606         >>> rb.config_toml = 'rustc = "rustc"\\n'
 607         >>> rb.program_config('rustc')
 608         'rustc'
 609         >>> rb.config_toml = ''
 610         >>> cargo_path = rb.program_config('cargo')
 611         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 612         ... "bin", "cargo")
 613         True
 614         """
 615         config = self.get_toml(program)
 616         if config:
 617             return os.path.expanduser(config)
 618         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 619             program, self.exe_suffix()))
 620
 621     @staticmethod
 622     def get_string(line):
 623         """Return the value between double quotes
 624
 625         >>> RustBuild.get_string('    "devel"   ')
 626         'devel'
 627         >>> RustBuild.get_string("    'devel'   ")
 628         'devel'
 629         >>> RustBuild.get_string('devel') is None
 630         True
 631         >>> RustBuild.get_string('    "devel   ')
 632         ''
 633         """
 634         start = line.find('"')
 635         if start != -1:
 636             end = start + 1 + line[start + 1:].find('"')
 637             return line[start + 1:end]
 638         start = line.find('\'')
 639         if start != -1:
 640             end = start + 1 + line[start + 1:].find('\'')
 641             return line[start + 1:end]
 642         return None
 643
 644     @staticmethod
 645     def exe_suffix():
 646         """Return a suffix for executables"""
 647         if sys.platform == 'win32':
 648             return '.exe'
 649         return ''
 650
 651     def bootstrap_binary(self):
 652         """Return the path of the bootstrap binary
 653
 654         >>> rb = RustBuild()
 655         >>> rb.build_dir = "build"
 656         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 657         ... "debug", "bootstrap")
 658         True
 659         """
 660         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 661
 662     def build_bootstrap(self):
 663         """Build bootstrap"""
 664         build_dir = os.path.join(self.build_dir, "bootstrap")
 665         if self.clean and os.path.exists(build_dir):
 666             shutil.rmtree(build_dir)
 667         env = os.environ.copy()
 668         env["RUSTC_BOOTSTRAP"] = '1'
 669         env["CARGO_TARGET_DIR"] = build_dir
 670         env["RUSTC"] = self.rustc()
 671         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 672             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 673             if "LD_LIBRARY_PATH" in env else ""
 674         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 675             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 676             if "DYLD_LIBRARY_PATH" in env else ""
 677         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 678             (os.pathsep + env["LIBRARY_PATH"]) \
 679             if "LIBRARY_PATH" in env else ""
 680         # preserve existing RUSTFLAGS
 681         env.setdefault("RUSTFLAGS", "")
 682         env["RUSTFLAGS"] += " -Cdebuginfo=2"
 683
 684         build_section = "target.{}".format(self.build_triple())
 685         target_features = []
 686         if self.get_toml("crt-static", build_section) == "true":
 687             target_features += ["+crt-static"]
 688         elif self.get_toml("crt-static", build_section) == "false":
 689             target_features += ["-crt-static"]
 690         if target_features:
 691             env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
 692         target_linker = self.get_toml("linker", build_section)
 693         if target_linker is not None:
 694             env["RUSTFLAGS"] += " -C linker=" + target_linker
 695         env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
 696         if self.get_toml("deny-warnings", "rust") != "false":
 697             env["RUSTFLAGS"] += " -Dwarnings"
 698
 699         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 700             os.pathsep + env["PATH"]
 701         if not os.path.isfile(self.cargo()):
 702             raise Exception("no cargo executable found at `{}`".format(
 703                 self.cargo()))
 704         args = [self.cargo(), "build", "--manifest-path",
 705                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 706         for _ in range(1, self.verbose):
 707             args.append("--verbose")
 708         if self.use_locked_deps:
 709             args.append("--locked")
 710         if self.use_vendored_sources:
 711             args.append("--frozen")
 712         run(args, env=env, verbose=self.verbose)
 713
 714     def build_triple(self):
 715         """Build triple as in LLVM"""
 716         config = self.get_toml('build')
 717         if config:
 718             return config
 719         return default_build_triple()
 720
 721     def check_submodule(self, module, slow_submodules):
 722         if not slow_submodules:
 723             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 724                                            cwd=os.path.join(self.rust_root, module),
 725                                            stdout=subprocess.PIPE)
 726             return checked_out
 727         else:
 728             return None
 729
 730     def update_submodule(self, module, checked_out, recorded_submodules):
 731         module_path = os.path.join(self.rust_root, module)
 732
 733         if checked_out is not None:
 734             default_encoding = sys.getdefaultencoding()
 735             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 736             if recorded_submodules[module] == checked_out:
 737                 return
 738
 739         print("Updating submodule", module)
 740
 741         run(["git", "submodule", "-q", "sync", module],
 742             cwd=self.rust_root, verbose=self.verbose)
 743         try:
 744             run(["git", "submodule", "update",
 745                  "--init", "--recursive", "--progress", module],
 746                 cwd=self.rust_root, verbose=self.verbose, exception=True)
 747         except RuntimeError:
 748             # Some versions of git don't support --progress.
 749             run(["git", "submodule", "update",
 750                  "--init", "--recursive", module],
 751                 cwd=self.rust_root, verbose=self.verbose)
 752         run(["git", "reset", "-q", "--hard"],
 753             cwd=module_path, verbose=self.verbose)
 754         run(["git", "clean", "-qdfx"],
 755             cwd=module_path, verbose=self.verbose)
 756
 757     def update_submodules(self):
 758         """Update submodules"""
 759         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 760                 self.get_toml('submodules') == "false":
 761             return
 762
 763         # check the existence of 'git' command
 764         try:
 765             subprocess.check_output(['git', '--version'])
 766         except (subprocess.CalledProcessError, OSError):
 767             print("error: `git` is not found, please make sure it's installed and in the path.")
 768             sys.exit(1)
 769
 770         slow_submodules = self.get_toml('fast-submodules') == "false"
 771         start_time = time()
 772         if slow_submodules:
 773             print('Unconditionally updating all submodules')
 774         else:
 775             print('Updating only changed submodules')
 776         default_encoding = sys.getdefaultencoding()
 777         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 778             ["git", "config", "--file",
 779              os.path.join(self.rust_root, ".gitmodules"),
 780              "--get-regexp", "path"]
 781         ).decode(default_encoding).splitlines()]
 782         filtered_submodules = []
 783         submodules_names = []
 784         for module in submodules:
 785             if module.endswith("llvm-project"):
 786                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 787                     continue
 788             check = self.check_submodule(module, slow_submodules)
 789             filtered_submodules.append((module, check))
 790             submodules_names.append(module)
 791         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 792                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 793         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 794         recorded_submodules = {}
 795         for data in recorded:
 796             data = data.split()
 797             recorded_submodules[data[3]] = data[2]
 798         for module in filtered_submodules:
 799             self.update_submodule(module[0], module[1], recorded_submodules)
 800         print("Submodules updated in %.2f seconds" % (time() - start_time))
 801
 802     def set_normal_environment(self):
 803         """Set download URL for normal environment"""
 804         if 'RUSTUP_DIST_SERVER' in os.environ:
 805             self._download_url = os.environ['RUSTUP_DIST_SERVER']
 806         else:
 807             self._download_url = 'https://static.rust-lang.org'
 808
 809     def set_dev_environment(self):
 810         """Set download URL for development environment"""
 811         if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
 812             self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
 813         else:
 814             self._download_url = 'https://dev-static.rust-lang.org'
 815
 816     def check_vendored_status(self):
 817         """Check that vendoring is configured properly"""
 818         vendor_dir = os.path.join(self.rust_root, 'vendor')
 819         if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
 820             if os.environ.get('USER') != os.environ['SUDO_USER']:
 821                 self.use_vendored_sources = True
 822                 print('info: looks like you are running this command under `sudo`')
 823                 print('      and so in order to preserve your $HOME this will now')
 824                 print('      use vendored sources by default.')
 825                 if not os.path.exists(vendor_dir):
 826                     print('error: vendoring required, but vendor directory does not exist.')
 827                     print('       Run `cargo vendor` without sudo to initialize the '
 828                         'vendor directory.')
 829                     raise Exception("{} not found".format(vendor_dir))
 830
 831         if self.use_vendored_sources:
 832             if not os.path.exists('.cargo'):
 833                 os.makedirs('.cargo')
 834             with output('.cargo/config') as cargo_config:
 835                 cargo_config.write(
 836                     "[source.crates-io]\n"
 837                     "replace-with = 'vendored-sources'\n"
 838                     "registry = 'https://example.com'\n"
 839                     "\n"
 840                     "[source.vendored-sources]\n"
 841                     "directory = '{}/vendor'\n"
 842                 .format(self.rust_root))
 843         else:
 844             if os.path.exists('.cargo'):
 845                 shutil.rmtree('.cargo')
 846
 847     def ensure_vendored(self):
 848         """Ensure that the vendored sources are available if needed"""
 849         vendor_dir = os.path.join(self.rust_root, 'vendor')
 850         # Note that this does not handle updating the vendored dependencies if
 851         # the rust git repository is updated. Normal development usually does
 852         # not use vendoring, so hopefully this isn't too much of a problem.
 853         if self.use_vendored_sources and not os.path.exists(vendor_dir):
 854             run([self.cargo(), "vendor"],
 855                 verbose=self.verbose, cwd=self.rust_root)
 856
 857
 858 def bootstrap(help_triggered):
 859     """Configure, fetch, build and run the initial bootstrap"""
 860
 861     # If the user is asking for help, let them know that the whole download-and-build
 862     # process has to happen before anything is printed out.
 863     if help_triggered:
 864         print("info: Downloading and building bootstrap before processing --help")
 865         print("      command. See src/bootstrap/README.md for help with common")
 866         print("      commands.")
 867
 868     parser = argparse.ArgumentParser(description='Build rust')
 869     parser.add_argument('--config')
 870     parser.add_argument('--build')
 871     parser.add_argument('--src')
 872     parser.add_argument('--clean', action='store_true')
 873     parser.add_argument('-v', '--verbose', action='count', default=0)
 874
 875     args = [a for a in sys.argv if a != '-h' and a != '--help']
 876     args, _ = parser.parse_known_args(args)
 877
 878     # Configure initial bootstrap
 879     build = RustBuild()
 880     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 881     build.verbose = args.verbose
 882     build.clean = args.clean
 883
 884     try:
 885         with open(args.config or 'config.toml') as config:
 886             build.config_toml = config.read()
 887     except (OSError, IOError):
 888         pass
 889
 890     config_verbose = build.get_toml('verbose', 'build')
 891     if config_verbose is not None:
 892         build.verbose = max(build.verbose, int(config_verbose))
 893
 894     build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
 895
 896     build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
 897
 898     build.check_vendored_status()
 899
 900     data = stage0_data(build.rust_root)
 901     build.date = data['date']
 902     build.rustc_channel = data['rustc']
 903     build.cargo_channel = data['cargo']
 904
 905     if "rustfmt" in data:
 906         build.rustfmt_channel = data['rustfmt']
 907
 908     if 'dev' in data:
 909         build.set_dev_environment()
 910     else:
 911         build.set_normal_environment()
 912
 913     build.update_submodules()
 914
 915     # Fetch/build the bootstrap
 916     build.build = args.build or build.build_triple()
 917     build.download_stage0()
 918     sys.stdout.flush()
 919     build.ensure_vendored()
 920     build.build_bootstrap()
 921     sys.stdout.flush()
 922
 923     # Run the bootstrap
 924     args = [build.bootstrap_binary()]
 925     args.extend(sys.argv[1:])
 926     env = os.environ.copy()
 927     env["BUILD"] = build.build
 928     env["SRC"] = build.rust_root
 929     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 930     env["BOOTSTRAP_PYTHON"] = sys.executable
 931     env["BUILD_DIR"] = build.build_dir
 932     env["RUSTC_BOOTSTRAP"] = '1'
 933     env["CARGO"] = build.cargo()
 934     env["RUSTC"] = build.rustc()
 935     if build.rustfmt():
 936         env["RUSTFMT"] = build.rustfmt()
 937     run(args, env=env, verbose=build.verbose)
 938
 939
 940 def main():
 941     """Entry point for the bootstrap process"""
 942     start_time = time()
 943
 944     # x.py help <cmd> ...
 945     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 946         sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
 947
 948     help_triggered = (
 949         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 950     try:
 951         bootstrap(help_triggered)
 952         if not help_triggered:
 953             print("Build completed successfully in {}".format(
 954                 format_build_time(time() - start_time)))
 955     except (SystemExit, KeyboardInterrupt) as error:
 956         if hasattr(error, 'code') and isinstance(error.code, int):
 957             exit_code = error.code
 958         else:
 959             exit_code = 1
 960             print(error)
 961         if not help_triggered:
 962             print("Build completed unsuccessfully in {}".format(
 963                 format_build_time(time() - start_time)))
 964         sys.exit(exit_code)
 965
 966
 967 if __name__ == '__main__':
 968     main()