src/bootstrap/bootstrap.py

   1 from __future__ import absolute_import, division, print_function
   2 import argparse
   3 import contextlib
   4 import datetime
   5 import hashlib
   6 import os
   7 import re
   8 import shutil
   9 import subprocess
  10 import sys
  11 import tarfile
  12 import tempfile
  13
  14 from time import time
  15
  16
  17 def get(url, path, verbose=False):
  18     suffix = '.sha256'
  19     sha_url = url + suffix
  20     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  21         temp_path = temp_file.name
  22     with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
  23         sha_path = sha_file.name
  24
  25     try:
  26         download(sha_path, sha_url, False, verbose)
  27         if os.path.exists(path):
  28             if verify(path, sha_path, False):
  29                 if verbose:
  30                     print("using already-download file", path)
  31                 return
  32             else:
  33                 if verbose:
  34                     print("ignoring already-download file",
  35                           path, "due to failed verification")
  36                 os.unlink(path)
  37         download(temp_path, url, True, verbose)
  38         if not verify(temp_path, sha_path, verbose):
  39             raise RuntimeError("failed verification")
  40         if verbose:
  41             print("moving {} to {}".format(temp_path, path))
  42         shutil.move(temp_path, path)
  43     finally:
  44         delete_if_present(sha_path, verbose)
  45         delete_if_present(temp_path, verbose)
  46
  47
  48 def delete_if_present(path, verbose):
  49     """Remove the given file if present"""
  50     if os.path.isfile(path):
  51         if verbose:
  52             print("removing", path)
  53         os.unlink(path)
  54
  55
  56 def download(path, url, probably_big, verbose):
  57     for _ in range(0, 4):
  58         try:
  59             _download(path, url, probably_big, verbose, True)
  60             return
  61         except RuntimeError:
  62             print("\nspurious failure, trying again")
  63     _download(path, url, probably_big, verbose, False)
  64
  65
  66 def _download(path, url, probably_big, verbose, exception):
  67     if probably_big or verbose:
  68         print("downloading {}".format(url))
  69     # see http://serverfault.com/questions/301128/how-to-download
  70     if sys.platform == 'win32':
  71         run(["PowerShell.exe", "/nologo", "-Command",
  72              "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
  73              "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
  74             verbose=verbose,
  75             exception=exception)
  76     else:
  77         if probably_big or verbose:
  78             option = "-#"
  79         else:
  80             option = "-s"
  81         run(["curl", option,
  82              "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
  83              "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
  84              "--retry", "3", "-Sf", "-o", path, url],
  85             verbose=verbose,
  86             exception=exception)
  87
  88
  89 def verify(path, sha_path, verbose):
  90     """Check if the sha256 sum of the given path is valid"""
  91     if verbose:
  92         print("verifying", path)
  93     with open(path, "rb") as source:
  94         found = hashlib.sha256(source.read()).hexdigest()
  95     with open(sha_path, "r") as sha256sum:
  96         expected = sha256sum.readline().split()[0]
  97     verified = found == expected
  98     if not verified:
  99         print("invalid checksum:\n"
 100               "    found:    {}\n"
 101               "    expected: {}".format(found, expected))
 102     return verified
 103
 104
 105 def unpack(tarball, dst, verbose=False, match=None):
 106     """Unpack the given tarball file"""
 107     print("extracting", tarball)
 108     fname = os.path.basename(tarball).replace(".tar.gz", "")
 109     with contextlib.closing(tarfile.open(tarball)) as tar:
 110         for member in tar.getnames():
 111             if "/" not in member:
 112                 continue
 113             name = member.replace(fname + "/", "", 1)
 114             if match is not None and not name.startswith(match):
 115                 continue
 116             name = name[len(match) + 1:]
 117
 118             dst_path = os.path.join(dst, name)
 119             if verbose:
 120                 print("  extracting", member)
 121             tar.extract(member, dst)
 122             src_path = os.path.join(dst, member)
 123             if os.path.isdir(src_path) and os.path.exists(dst_path):
 124                 continue
 125             shutil.move(src_path, dst_path)
 126     shutil.rmtree(os.path.join(dst, fname))
 127
 128
 129 def run(args, verbose=False, exception=False, **kwargs):
 130     """Run a child program in a new process"""
 131     if verbose:
 132         print("running: " + ' '.join(args))
 133     sys.stdout.flush()
 134     # Use Popen here instead of call() as it apparently allows powershell on
 135     # Windows to not lock up waiting for input presumably.
 136     ret = subprocess.Popen(args, **kwargs)
 137     code = ret.wait()
 138     if code != 0:
 139         err = "failed to run: " + ' '.join(args)
 140         if verbose or exception:
 141             raise RuntimeError(err)
 142         sys.exit(err)
 143
 144
 145 def stage0_data(rust_root):
 146     """Build a dictionary from stage0.txt"""
 147     nightlies = os.path.join(rust_root, "src/stage0.txt")
 148     with open(nightlies, 'r') as nightlies:
 149         lines = [line.rstrip() for line in nightlies
 150                  if not line.startswith("#")]
 151         return dict([line.split(": ", 1) for line in lines if line])
 152
 153
 154 def format_build_time(duration):
 155     """Return a nicer format for build time
 156
 157     >>> format_build_time('300')
 158     '0:05:00'
 159     """
 160     return str(datetime.timedelta(seconds=int(duration)))
 161
 162
 163 def default_build_triple():
 164     """Build triple as in LLVM"""
 165     default_encoding = sys.getdefaultencoding()
 166     try:
 167         ostype = subprocess.check_output(
 168             ['uname', '-s']).strip().decode(default_encoding)
 169         cputype = subprocess.check_output(
 170             ['uname', '-m']).strip().decode(default_encoding)
 171     except (subprocess.CalledProcessError, OSError):
 172         if sys.platform == 'win32':
 173             return 'x86_64-pc-windows-msvc'
 174         err = "uname not found"
 175         sys.exit(err)
 176
 177     # The goal here is to come up with the same triple as LLVM would,
 178     # at least for the subset of platforms we're willing to target.
 179     ostype_mapper = {
 180         'Bitrig': 'unknown-bitrig',
 181         'Darwin': 'apple-darwin',
 182         'DragonFly': 'unknown-dragonfly',
 183         'FreeBSD': 'unknown-freebsd',
 184         'Haiku': 'unknown-haiku',
 185         'NetBSD': 'unknown-netbsd',
 186         'OpenBSD': 'unknown-openbsd'
 187     }
 188
 189     # Consider the direct transformation first and then the special cases
 190     if ostype in ostype_mapper:
 191         ostype = ostype_mapper[ostype]
 192     elif ostype == 'Linux':
 193         os_from_sp = subprocess.check_output(
 194             ['uname', '-o']).strip().decode(default_encoding)
 195         if os_from_sp == 'Android':
 196             ostype = 'linux-android'
 197         else:
 198             ostype = 'unknown-linux-gnu'
 199     elif ostype == 'SunOS':
 200         ostype = 'sun-solaris'
 201         # On Solaris, uname -m will return a machine classification instead
 202         # of a cpu type, so uname -p is recommended instead.  However, the
 203         # output from that option is too generic for our purposes (it will
 204         # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 205         # must be used instead.
 206         try:
 207             cputype = subprocess.check_output(
 208                 ['isainfo', '-k']).strip().decode(default_encoding)
 209         except (subprocess.CalledProcessError, OSError):
 210             err = "isainfo not found"
 211             sys.exit(err)
 212     elif ostype.startswith('MINGW'):
 213         # msys' `uname` does not print gcc configuration, but prints msys
 214         # configuration. so we cannot believe `uname -m`:
 215         # msys1 is always i686 and msys2 is always x86_64.
 216         # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 217         # MINGW64 on x86_64.
 218         ostype = 'pc-windows-gnu'
 219         cputype = 'i686'
 220         if os.environ.get('MSYSTEM') == 'MINGW64':
 221             cputype = 'x86_64'
 222     elif ostype.startswith('MSYS'):
 223         ostype = 'pc-windows-gnu'
 224     elif ostype.startswith('CYGWIN_NT'):
 225         cputype = 'i686'
 226         if ostype.endswith('WOW64'):
 227             cputype = 'x86_64'
 228         ostype = 'pc-windows-gnu'
 229     else:
 230         err = "unknown OS type: {}".format(ostype)
 231         sys.exit(err)
 232
 233     if cputype == 'powerpc' and ostype == 'unknown-freebsd':
 234         cputype = subprocess.check_output(
 235               ['uname', '-p']).strip().decode(default_encoding)
 236     cputype_mapper = {
 237         'BePC': 'i686',
 238         'aarch64': 'aarch64',
 239         'amd64': 'x86_64',
 240         'arm64': 'aarch64',
 241         'i386': 'i686',
 242         'i486': 'i686',
 243         'i686': 'i686',
 244         'i786': 'i686',
 245         'powerpc': 'powerpc',
 246         'powerpc64': 'powerpc64',
 247         'powerpc64le': 'powerpc64le',
 248         'ppc': 'powerpc',
 249         'ppc64': 'powerpc64',
 250         'ppc64le': 'powerpc64le',
 251         's390x': 's390x',
 252         'x64': 'x86_64',
 253         'x86': 'i686',
 254         'x86-64': 'x86_64',
 255         'x86_64': 'x86_64'
 256     }
 257
 258     # Consider the direct transformation first and then the special cases
 259     if cputype in cputype_mapper:
 260         cputype = cputype_mapper[cputype]
 261     elif cputype in {'xscale', 'arm'}:
 262         cputype = 'arm'
 263         if ostype == 'linux-android':
 264             ostype = 'linux-androideabi'
 265     elif cputype == 'armv6l':
 266         cputype = 'arm'
 267         if ostype == 'linux-android':
 268             ostype = 'linux-androideabi'
 269         else:
 270             ostype += 'eabihf'
 271     elif cputype in {'armv7l', 'armv8l'}:
 272         cputype = 'armv7'
 273         if ostype == 'linux-android':
 274             ostype = 'linux-androideabi'
 275         else:
 276             ostype += 'eabihf'
 277     elif cputype == 'mips':
 278         if sys.byteorder == 'big':
 279             cputype = 'mips'
 280         elif sys.byteorder == 'little':
 281             cputype = 'mipsel'
 282         else:
 283             raise ValueError("unknown byteorder: {}".format(sys.byteorder))
 284     elif cputype == 'mips64':
 285         if sys.byteorder == 'big':
 286             cputype = 'mips64'
 287         elif sys.byteorder == 'little':
 288             cputype = 'mips64el'
 289         else:
 290             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
 291         # only the n64 ABI is supported, indicate it
 292         ostype += 'abi64'
 293     elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
 294         pass
 295     else:
 296         err = "unknown cpu type: {}".format(cputype)
 297         sys.exit(err)
 298
 299     return "{}-{}".format(cputype, ostype)
 300
 301
 302 @contextlib.contextmanager
 303 def output(filepath):
 304     tmp = filepath + '.tmp'
 305     with open(tmp, 'w') as f:
 306         yield f
 307     try:
 308         os.remove(filepath)  # PermissionError/OSError on Win32 if in use
 309         os.rename(tmp, filepath)
 310     except OSError:
 311         shutil.copy2(tmp, filepath)
 312         os.remove(tmp)
 313
 314
 315 class RustBuild(object):
 316     """Provide all the methods required to build Rust"""
 317     def __init__(self):
 318         self.cargo_channel = ''
 319         self.date = ''
 320         self._download_url = 'https://static.rust-lang.org'
 321         self.rustc_channel = ''
 322         self.build = ''
 323         self.build_dir = os.path.join(os.getcwd(), "build")
 324         self.clean = False
 325         self.config_toml = ''
 326         self.rust_root = ''
 327         self.use_locked_deps = ''
 328         self.use_vendored_sources = ''
 329         self.verbose = False
 330
 331     def download_stage0(self):
 332         """Fetch the build system for Rust, written in Rust
 333
 334         This method will build a cache directory, then it will fetch the
 335         tarball which has the stage0 compiler used to then bootstrap the Rust
 336         compiler itself.
 337
 338         Each downloaded tarball is extracted, after that, the script
 339         will move all the content to the right place.
 340         """
 341         rustc_channel = self.rustc_channel
 342         cargo_channel = self.cargo_channel
 343
 344         if self.rustc().startswith(self.bin_root()) and \
 345                 (not os.path.exists(self.rustc()) or
 346                  self.program_out_of_date(self.rustc_stamp())):
 347             if os.path.exists(self.bin_root()):
 348                 shutil.rmtree(self.bin_root())
 349             filename = "rust-std-{}-{}.tar.gz".format(
 350                 rustc_channel, self.build)
 351             pattern = "rust-std-{}".format(self.build)
 352             self._download_stage0_helper(filename, pattern)
 353
 354             filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
 355             self._download_stage0_helper(filename, "rustc")
 356             self.fix_executable("{}/bin/rustc".format(self.bin_root()))
 357             self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
 358             with output(self.rustc_stamp()) as rust_stamp:
 359                 rust_stamp.write(self.date)
 360
 361             # This is required so that we don't mix incompatible MinGW
 362             # libraries/binaries that are included in rust-std with
 363             # the system MinGW ones.
 364             if "pc-windows-gnu" in self.build:
 365                 filename = "rust-mingw-{}-{}.tar.gz".format(
 366                     rustc_channel, self.build)
 367                 self._download_stage0_helper(filename, "rust-mingw")
 368
 369         if self.cargo().startswith(self.bin_root()) and \
 370                 (not os.path.exists(self.cargo()) or
 371                  self.program_out_of_date(self.cargo_stamp())):
 372             filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
 373             self._download_stage0_helper(filename, "cargo")
 374             self.fix_executable("{}/bin/cargo".format(self.bin_root()))
 375             with output(self.cargo_stamp()) as cargo_stamp:
 376                 cargo_stamp.write(self.date)
 377
 378     def _download_stage0_helper(self, filename, pattern):
 379         cache_dst = os.path.join(self.build_dir, "cache")
 380         rustc_cache = os.path.join(cache_dst, self.date)
 381         if not os.path.exists(rustc_cache):
 382             os.makedirs(rustc_cache)
 383
 384         url = "{}/dist/{}".format(self._download_url, self.date)
 385         tarball = os.path.join(rustc_cache, filename)
 386         if not os.path.exists(tarball):
 387             get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 388         unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
 389
 390     @staticmethod
 391     def fix_executable(fname):
 392         """Modifies the interpreter section of 'fname' to fix the dynamic linker
 393
 394         This method is only required on NixOS and uses the PatchELF utility to
 395         change the dynamic linker of ELF executables.
 396
 397         Please see https://nixos.org/patchelf.html for more information
 398         """
 399         default_encoding = sys.getdefaultencoding()
 400         try:
 401             ostype = subprocess.check_output(
 402                 ['uname', '-s']).strip().decode(default_encoding)
 403         except subprocess.CalledProcessError:
 404             return
 405         except OSError as reason:
 406             if getattr(reason, 'winerror', None) is not None:
 407                 return
 408             raise reason
 409
 410         if ostype != "Linux":
 411             return
 412
 413         if not os.path.exists("/etc/NIXOS"):
 414             return
 415         if os.path.exists("/lib"):
 416             return
 417
 418         # At this point we're pretty sure the user is running NixOS
 419         nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
 420         print(nix_os_msg, fname)
 421
 422         try:
 423             interpreter = subprocess.check_output(
 424                 ["patchelf", "--print-interpreter", fname])
 425             interpreter = interpreter.strip().decode(default_encoding)
 426         except subprocess.CalledProcessError as reason:
 427             print("warning: failed to call patchelf:", reason)
 428             return
 429
 430         loader = interpreter.split("/")[-1]
 431
 432         try:
 433             ldd_output = subprocess.check_output(
 434                 ['ldd', '/run/current-system/sw/bin/sh'])
 435             ldd_output = ldd_output.strip().decode(default_encoding)
 436         except subprocess.CalledProcessError as reason:
 437             print("warning: unable to call ldd:", reason)
 438             return
 439
 440         for line in ldd_output.splitlines():
 441             libname = line.split()[0]
 442             if libname.endswith(loader):
 443                 loader_path = libname[:len(libname) - len(loader)]
 444                 break
 445         else:
 446             print("warning: unable to find the path to the dynamic linker")
 447             return
 448
 449         correct_interpreter = loader_path + loader
 450
 451         try:
 452             subprocess.check_output(
 453                 ["patchelf", "--set-interpreter", correct_interpreter, fname])
 454         except subprocess.CalledProcessError as reason:
 455             print("warning: failed to call patchelf:", reason)
 456             return
 457
 458     def rustc_stamp(self):
 459         """Return the path for .rustc-stamp
 460
 461         >>> rb = RustBuild()
 462         >>> rb.build_dir = "build"
 463         >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
 464         True
 465         """
 466         return os.path.join(self.bin_root(), '.rustc-stamp')
 467
 468     def cargo_stamp(self):
 469         """Return the path for .cargo-stamp
 470
 471         >>> rb = RustBuild()
 472         >>> rb.build_dir = "build"
 473         >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
 474         True
 475         """
 476         return os.path.join(self.bin_root(), '.cargo-stamp')
 477
 478     def program_out_of_date(self, stamp_path):
 479         """Check if the given program stamp is out of date"""
 480         if not os.path.exists(stamp_path) or self.clean:
 481             return True
 482         with open(stamp_path, 'r') as stamp:
 483             return self.date != stamp.read()
 484
 485     def bin_root(self):
 486         """Return the binary root directory
 487
 488         >>> rb = RustBuild()
 489         >>> rb.build_dir = "build"
 490         >>> rb.bin_root() == os.path.join("build", "stage0")
 491         True
 492
 493         When the 'build' property is given should be a nested directory:
 494
 495         >>> rb.build = "devel"
 496         >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
 497         True
 498         """
 499         return os.path.join(self.build_dir, self.build, "stage0")
 500
 501     def get_toml(self, key, section=None):
 502         """Returns the value of the given key in config.toml, otherwise returns None
 503
 504         >>> rb = RustBuild()
 505         >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
 506         >>> rb.get_toml("key2")
 507         'value2'
 508
 509         If the key does not exists, the result is None:
 510
 511         >>> rb.get_toml("key3") is None
 512         True
 513
 514         Optionally also matches the section the key appears in
 515
 516         >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
 517         >>> rb.get_toml('key', 'a')
 518         'value1'
 519         >>> rb.get_toml('key', 'b')
 520         'value2'
 521         >>> rb.get_toml('key', 'c') is None
 522         True
 523         """
 524
 525         cur_section = None
 526         for line in self.config_toml.splitlines():
 527             section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
 528             if section_match is not None:
 529                 cur_section = section_match.group(1)
 530
 531             match = re.match(r'^{}\s*=(.*)$'.format(key), line)
 532             if match is not None:
 533                 value = match.group(1)
 534                 if section is None or section == cur_section:
 535                     return self.get_string(value) or value.strip()
 536         return None
 537
 538     def cargo(self):
 539         """Return config path for cargo"""
 540         return self.program_config('cargo')
 541
 542     def rustc(self):
 543         """Return config path for rustc"""
 544         return self.program_config('rustc')
 545
 546     def program_config(self, program):
 547         """Return config path for the given program
 548
 549         >>> rb = RustBuild()
 550         >>> rb.config_toml = 'rustc = "rustc"\\n'
 551         >>> rb.program_config('rustc')
 552         'rustc'
 553         >>> rb.config_toml = ''
 554         >>> cargo_path = rb.program_config('cargo')
 555         >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
 556         ... "bin", "cargo")
 557         True
 558         """
 559         config = self.get_toml(program)
 560         if config:
 561             return os.path.expanduser(config)
 562         return os.path.join(self.bin_root(), "bin", "{}{}".format(
 563             program, self.exe_suffix()))
 564
 565     @staticmethod
 566     def get_string(line):
 567         """Return the value between double quotes
 568
 569         >>> RustBuild.get_string('    "devel"   ')
 570         'devel'
 571         """
 572         start = line.find('"')
 573         if start != -1:
 574             end = start + 1 + line[start + 1:].find('"')
 575             return line[start + 1:end]
 576         start = line.find('\'')
 577         if start != -1:
 578             end = start + 1 + line[start + 1:].find('\'')
 579             return line[start + 1:end]
 580         return None
 581
 582     @staticmethod
 583     def exe_suffix():
 584         """Return a suffix for executables"""
 585         if sys.platform == 'win32':
 586             return '.exe'
 587         return ''
 588
 589     def bootstrap_binary(self):
 590         """Return the path of the bootstrap binary
 591
 592         >>> rb = RustBuild()
 593         >>> rb.build_dir = "build"
 594         >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
 595         ... "debug", "bootstrap")
 596         True
 597         """
 598         return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
 599
 600     def build_bootstrap(self):
 601         """Build bootstrap"""
 602         build_dir = os.path.join(self.build_dir, "bootstrap")
 603         if self.clean and os.path.exists(build_dir):
 604             shutil.rmtree(build_dir)
 605         env = os.environ.copy()
 606         env["RUSTC_BOOTSTRAP"] = '1'
 607         env["CARGO_TARGET_DIR"] = build_dir
 608         env["RUSTC"] = self.rustc()
 609         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 610             (os.pathsep + env["LD_LIBRARY_PATH"]) \
 611             if "LD_LIBRARY_PATH" in env else ""
 612         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 613             (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 614             if "DYLD_LIBRARY_PATH" in env else ""
 615         env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 616             (os.pathsep + env["LIBRARY_PATH"]) \
 617             if "LIBRARY_PATH" in env else ""
 618         env["RUSTFLAGS"] = "-Cdebuginfo=2 "
 619
 620         build_section = "target.{}".format(self.build_triple())
 621         target_features = []
 622         if self.get_toml("crt-static", build_section) == "true":
 623             target_features += ["+crt-static"]
 624         elif self.get_toml("crt-static", build_section) == "false":
 625             target_features += ["-crt-static"]
 626         if target_features:
 627             env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
 628         target_linker = self.get_toml("linker", build_section)
 629         if target_linker is not None:
 630             env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
 631
 632         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 633             os.pathsep + env["PATH"]
 634         if not os.path.isfile(self.cargo()):
 635             raise Exception("no cargo executable found at `{}`".format(
 636                 self.cargo()))
 637         args = [self.cargo(), "build", "--manifest-path",
 638                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 639         for _ in range(1, self.verbose):
 640             args.append("--verbose")
 641         if self.use_locked_deps:
 642             args.append("--locked")
 643         if self.use_vendored_sources:
 644             args.append("--frozen")
 645         run(args, env=env, verbose=self.verbose)
 646
 647     def build_triple(self):
 648         """Build triple as in LLVM"""
 649         config = self.get_toml('build')
 650         if config:
 651             return config
 652         return default_build_triple()
 653
 654     def check_submodule(self, module, slow_submodules):
 655         if not slow_submodules:
 656             checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
 657                                            cwd=os.path.join(self.rust_root, module),
 658                                            stdout=subprocess.PIPE)
 659             return checked_out
 660         else:
 661             return None
 662
 663     def update_submodule(self, module, checked_out, recorded_submodules):
 664         module_path = os.path.join(self.rust_root, module)
 665
 666         if checked_out != None:
 667             default_encoding = sys.getdefaultencoding()
 668             checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
 669             if recorded_submodules[module] == checked_out:
 670                 return
 671
 672         print("Updating submodule", module)
 673
 674         run(["git", "submodule", "-q", "sync", module],
 675             cwd=self.rust_root, verbose=self.verbose)
 676         run(["git", "submodule", "update",
 677             "--init", "--recursive", "--progress", module],
 678             cwd=self.rust_root, verbose=self.verbose)
 679         run(["git", "reset", "-q", "--hard"],
 680             cwd=module_path, verbose=self.verbose)
 681         run(["git", "clean", "-qdfx"],
 682             cwd=module_path, verbose=self.verbose)
 683
 684     def update_submodules(self):
 685         """Update submodules"""
 686         if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
 687                 self.get_toml('submodules') == "false":
 688             return
 689         slow_submodules = self.get_toml('fast-submodules') == "false"
 690         start_time = time()
 691         if slow_submodules:
 692             print('Unconditionally updating all submodules')
 693         else:
 694             print('Updating only changed submodules')
 695         default_encoding = sys.getdefaultencoding()
 696         submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
 697             ["git", "config", "--file",
 698              os.path.join(self.rust_root, ".gitmodules"),
 699              "--get-regexp", "path"]
 700         ).decode(default_encoding).splitlines()]
 701         filtered_submodules = []
 702         submodules_names = []
 703         for module in submodules:
 704             if module.endswith("llvm-project"):
 705                 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
 706                     continue
 707             if module.endswith("llvm-emscripten"):
 708                 backends = self.get_toml('codegen-backends')
 709                 if backends is None or not 'emscripten' in backends:
 710                     continue
 711             check = self.check_submodule(module, slow_submodules)
 712             filtered_submodules.append((module, check))
 713             submodules_names.append(module)
 714         recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
 715                                     cwd=self.rust_root, stdout=subprocess.PIPE)
 716         recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
 717         recorded_submodules = {}
 718         for data in recorded:
 719             data = data.split()
 720             recorded_submodules[data[3]] = data[2]
 721         for module in filtered_submodules:
 722             self.update_submodule(module[0], module[1], recorded_submodules)
 723         print("Submodules updated in %.2f seconds" % (time() - start_time))
 724
 725     def set_dev_environment(self):
 726         """Set download URL for development environment"""
 727         self._download_url = 'https://dev-static.rust-lang.org'
 728
 729
 730 def bootstrap(help_triggered):
 731     """Configure, fetch, build and run the initial bootstrap"""
 732
 733     # If the user is asking for help, let them know that the whole download-and-build
 734     # process has to happen before anything is printed out.
 735     if help_triggered:
 736         print("info: Downloading and building bootstrap before processing --help")
 737         print("      command. See src/bootstrap/README.md for help with common")
 738         print("      commands.")
 739
 740     parser = argparse.ArgumentParser(description='Build rust')
 741     parser.add_argument('--config')
 742     parser.add_argument('--build')
 743     parser.add_argument('--src')
 744     parser.add_argument('--clean', action='store_true')
 745     parser.add_argument('-v', '--verbose', action='count', default=0)
 746
 747     args = [a for a in sys.argv if a != '-h' and a != '--help']
 748     args, _ = parser.parse_known_args(args)
 749
 750     # Configure initial bootstrap
 751     build = RustBuild()
 752     build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
 753     build.verbose = args.verbose
 754     build.clean = args.clean
 755
 756     try:
 757         with open(args.config or 'config.toml') as config:
 758             build.config_toml = config.read()
 759     except (OSError, IOError):
 760         pass
 761
 762     match = re.search(r'\nverbose = (\d+)', build.config_toml)
 763     if match is not None:
 764         build.verbose = max(build.verbose, int(match.group(1)))
 765
 766     build.use_vendored_sources = '\nvendor = true' in build.config_toml
 767
 768     build.use_locked_deps = '\nlocked-deps = true' in build.config_toml
 769
 770     if 'SUDO_USER' in os.environ and not build.use_vendored_sources:
 771         if os.environ.get('USER') != os.environ['SUDO_USER']:
 772             build.use_vendored_sources = True
 773             print('info: looks like you are running this command under `sudo`')
 774             print('      and so in order to preserve your $HOME this will now')
 775             print('      use vendored sources by default. Note that if this')
 776             print('      does not work you should run a normal build first')
 777             print('      before running a command like `sudo ./x.py install`')
 778
 779     if build.use_vendored_sources:
 780         if not os.path.exists('.cargo'):
 781             os.makedirs('.cargo')
 782         with output('.cargo/config') as cargo_config:
 783             cargo_config.write("""
 784                 [source.crates-io]
 785                 replace-with = 'vendored-sources'
 786                 registry = 'https://example.com'
 787
 788                 [source.vendored-sources]
 789                 directory = '{}/vendor'
 790             """.format(build.rust_root))
 791     else:
 792         if os.path.exists('.cargo'):
 793             shutil.rmtree('.cargo')
 794
 795     data = stage0_data(build.rust_root)
 796     build.date = data['date']
 797     build.rustc_channel = data['rustc']
 798     build.cargo_channel = data['cargo']
 799
 800     if 'dev' in data:
 801         build.set_dev_environment()
 802
 803     build.update_submodules()
 804
 805     # Fetch/build the bootstrap
 806     build.build = args.build or build.build_triple()
 807     build.download_stage0()
 808     sys.stdout.flush()
 809     build.build_bootstrap()
 810     sys.stdout.flush()
 811
 812     # Run the bootstrap
 813     args = [build.bootstrap_binary()]
 814     args.extend(sys.argv[1:])
 815     env = os.environ.copy()
 816     env["BUILD"] = build.build
 817     env["SRC"] = build.rust_root
 818     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 819     env["BOOTSTRAP_PYTHON"] = sys.executable
 820     env["BUILD_DIR"] = build.build_dir
 821     env["RUSTC_BOOTSTRAP"] = '1'
 822     env["CARGO"] = build.cargo()
 823     env["RUSTC"] = build.rustc()
 824     run(args, env=env, verbose=build.verbose)
 825
 826
 827 def main():
 828     """Entry point for the bootstrap process"""
 829     start_time = time()
 830
 831     # x.py help <cmd> ...
 832     if len(sys.argv) > 1 and sys.argv[1] == 'help':
 833         sys.argv = sys.argv[:1] + [sys.argv[2], '-h'] + sys.argv[3:]
 834
 835     help_triggered = (
 836         '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
 837     try:
 838         bootstrap(help_triggered)
 839         if not help_triggered:
 840             print("Build completed successfully in {}".format(
 841                 format_build_time(time() - start_time)))
 842     except (SystemExit, KeyboardInterrupt) as error:
 843         if hasattr(error, 'code') and isinstance(error.code, int):
 844             exit_code = error.code
 845         else:
 846             exit_code = 1
 847             print(error)
 848         if not help_triggered:
 849             print("Build completed unsuccessfully in {}".format(
 850                 format_build_time(time() - start_time)))
 851         sys.exit(exit_code)
 852
 853
 854 if __name__ == '__main__':
 855     main()