src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 from __future__ import print_function
  12 import argparse
  13 import contextlib
  14 import datetime
  15 import hashlib
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tarfile
  21 import tempfile
  22
  23 from time import time
  24
  25
  26 def get(url, path, verbose=False):
  27     sha_url = url + ".sha256"
  28     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  29         temp_path = temp_file.name
  30     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  31         sha_path = sha_file.name
  32
  33     try:
  34         download(sha_path, sha_url, False, verbose)
  35         if os.path.exists(path):
  36             if verify(path, sha_path, False):
  37                 if verbose:
  38                     print("using already-download file " + path)
  39                 return
  40             else:
  41                 if verbose:
  42                     print("ignoring already-download file " + path + " due to failed verification")
  43                 os.unlink(path)
  44         download(temp_path, url, True, verbose)
  45         if not verify(temp_path, sha_path, verbose):
  46             raise RuntimeError("failed verification")
  47         if verbose:
  48             print("moving {} to {}".format(temp_path, path))
  49         shutil.move(temp_path, path)
  50     finally:
  51         delete_if_present(sha_path, verbose)
  52         delete_if_present(temp_path, verbose)
  53
  54
  55 def delete_if_present(path, verbose):
  56     if os.path.isfile(path):
  57         if verbose:
  58             print("removing " + path)
  59         os.unlink(path)
  60
  61
  62 def download(path, url, probably_big, verbose):
  63     for x in range(0, 4):
  64         try:
  65             _download(path, url, probably_big, verbose, True)
  66             return
  67         except RuntimeError:
  68             print("\nspurious failure, trying again")
  69     _download(path, url, probably_big, verbose, False)
  70
  71
  72 def _download(path, url, probably_big, verbose, exception):
  73     if probably_big or verbose:
  74         print("downloading {}".format(url))
  75     # see http://serverfault.com/questions/301128/how-to-download
  76     if sys.platform == 'win32':
  77         run(["PowerShell.exe", "/nologo", "-Command",
  78              "(New-Object System.Net.WebClient)"
  79              ".DownloadFile('{}', '{}')".format(url, path)],
  80             verbose=verbose,
  81             exception=exception)
  82     else:
  83         if probably_big or verbose:
  84             option = "-#"
  85         else:
  86             option = "-s"
  87         run(["curl", option, "--retry", "3", "-Sf", "-o", path, url],
  88             verbose=verbose,
  89             exception=exception)
  90
  91
  92 def verify(path, sha_path, verbose):
  93     if verbose:
  94         print("verifying " + path)
  95     with open(path, "rb") as f:
  96         found = hashlib.sha256(f.read()).hexdigest()
  97     with open(sha_path, "r") as f:
  98         expected = f.readline().split()[0]
  99     verified = found == expected
 100     if not verified:
 101         print("invalid checksum:\n"
 102                "    found:    {}\n"
 103                "    expected: {}".format(found, expected))
 104     return verified
 105
 106
 107 def unpack(tarball, dst, verbose=False, match=None):
 108     print("extracting " + tarball)
 109     fname = os.path.basename(tarball).replace(".tar.gz", "")
 110     with contextlib.closing(tarfile.open(tarball)) as tar:
 111         for p in tar.getnames():
 112             if "/" not in p:
 113                 continue
 114             name = p.replace(fname + "/", "", 1)
 115             if match is not None and not name.startswith(match):
 116                 continue
 117             name = name[len(match) + 1:]
 118
 119             fp = os.path.join(dst, name)
 120             if verbose:
 121                 print("  extracting " + p)
 122             tar.extract(p, dst)
 123             tp = os.path.join(dst, p)
 124             if os.path.isdir(tp) and os.path.exists(fp):
 125                 continue
 126             shutil.move(tp, fp)
 127     shutil.rmtree(os.path.join(dst, fname))
 128
 129 def run(args, verbose=False, exception=False):
 130     if verbose:
 131         print("running: " + ' '.join(args))
 132     sys.stdout.flush()
 133     # Use Popen here instead of call() as it apparently allows powershell on
 134     # Windows to not lock up waiting for input presumably.
 135     ret = subprocess.Popen(args)
 136     code = ret.wait()
 137     if code != 0:
 138         err = "failed to run: " + ' '.join(args)
 139         if verbose or exception:
 140             raise RuntimeError(err)
 141         sys.exit(err)
 142
 143 def stage0_data(rust_root):
 144     nightlies = os.path.join(rust_root, "src/stage0.txt")
 145     data = {}
 146     with open(nightlies, 'r') as nightlies:
 147         for line in nightlies:
 148             line = line.rstrip()  # Strip newline character, '\n'
 149             if line.startswith("#") or line == '':
 150                 continue
 151             a, b = line.split(": ", 1)
 152             data[a] = b
 153     return data
 154
 155 def format_build_time(duration):
 156     return str(datetime.timedelta(seconds=int(duration)))
 157
 158
 159 class RustBuild(object):
 160     def download_stage0(self):
 161         cache_dst = os.path.join(self.build_dir, "cache")
 162         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 163         cargo_cache = os.path.join(cache_dst, self.stage0_cargo_rev())
 164         if not os.path.exists(rustc_cache):
 165             os.makedirs(rustc_cache)
 166         if not os.path.exists(cargo_cache):
 167             os.makedirs(cargo_cache)
 168
 169         if self.rustc().startswith(self.bin_root()) and \
 170                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 171             self.print_what_it_means_to_bootstrap()
 172             if os.path.exists(self.bin_root()):
 173                 shutil.rmtree(self.bin_root())
 174             channel = self.stage0_rustc_channel()
 175             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 176             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 177             tarball = os.path.join(rustc_cache, filename)
 178             if not os.path.exists(tarball):
 179                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 180             unpack(tarball, self.bin_root(),
 181                    match="rust-std-" + self.build,
 182                    verbose=self.verbose)
 183
 184             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 185             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 186             tarball = os.path.join(rustc_cache, filename)
 187             if not os.path.exists(tarball):
 188                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 189             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 190             self.fix_executable(self.bin_root() + "/bin/rustc")
 191             self.fix_executable(self.bin_root() + "/bin/rustdoc")
 192             with open(self.rustc_stamp(), 'w') as f:
 193                 f.write(self.stage0_rustc_date())
 194
 195         if self.cargo().startswith(self.bin_root()) and \
 196                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 197             self.print_what_it_means_to_bootstrap()
 198             filename = "cargo-nightly-{}.tar.gz".format(self.build)
 199             url = "https://s3.amazonaws.com/rust-lang-ci/cargo-builds/" + self.stage0_cargo_rev()
 200             tarball = os.path.join(cargo_cache, filename)
 201             if not os.path.exists(tarball):
 202                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 203             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 204             self.fix_executable(self.bin_root() + "/bin/cargo")
 205             with open(self.cargo_stamp(), 'w') as f:
 206                 f.write(self.stage0_cargo_rev())
 207
 208     def fix_executable(self, fname):
 209         # If we're on NixOS we need to change the path to the dynamic loader
 210
 211         default_encoding = sys.getdefaultencoding()
 212         try:
 213             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 214         except (subprocess.CalledProcessError, WindowsError):
 215             return
 216
 217         if ostype != "Linux":
 218             return
 219
 220         if not os.path.exists("/etc/NIXOS"):
 221             return
 222         if os.path.exists("/lib"):
 223             return
 224
 225         # At this point we're pretty sure the user is running NixOS
 226         print("info: you seem to be running NixOS. Attempting to patch " + fname)
 227
 228         try:
 229             interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
 230             interpreter = interpreter.strip().decode(default_encoding)
 231         except subprocess.CalledProcessError as e:
 232             print("warning: failed to call patchelf: %s" % e)
 233             return
 234
 235         loader = interpreter.split("/")[-1]
 236
 237         try:
 238             ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
 239             ldd_output = ldd_output.strip().decode(default_encoding)
 240         except subprocess.CalledProcessError as e:
 241             print("warning: unable to call ldd: %s" % e)
 242             return
 243
 244         for line in ldd_output.splitlines():
 245             libname = line.split()[0]
 246             if libname.endswith(loader):
 247                 loader_path = libname[:len(libname) - len(loader)]
 248                 break
 249         else:
 250             print("warning: unable to find the path to the dynamic linker")
 251             return
 252
 253         correct_interpreter = loader_path + loader
 254
 255         try:
 256             subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
 257         except subprocess.CalledProcessError as e:
 258             print("warning: failed to call patchelf: %s" % e)
 259             return
 260
 261     def stage0_cargo_rev(self):
 262         return self._cargo_rev
 263
 264     def stage0_rustc_date(self):
 265         return self._rustc_date
 266
 267     def stage0_rustc_channel(self):
 268         return self._rustc_channel
 269
 270     def rustc_stamp(self):
 271         return os.path.join(self.bin_root(), '.rustc-stamp')
 272
 273     def cargo_stamp(self):
 274         return os.path.join(self.bin_root(), '.cargo-stamp')
 275
 276     def rustc_out_of_date(self):
 277         if not os.path.exists(self.rustc_stamp()) or self.clean:
 278             return True
 279         with open(self.rustc_stamp(), 'r') as f:
 280             return self.stage0_rustc_date() != f.read()
 281
 282     def cargo_out_of_date(self):
 283         if not os.path.exists(self.cargo_stamp()) or self.clean:
 284             return True
 285         with open(self.cargo_stamp(), 'r') as f:
 286             return self.stage0_cargo_rev() != f.read()
 287
 288     def bin_root(self):
 289         return os.path.join(self.build_dir, self.build, "stage0")
 290
 291     def get_toml(self, key):
 292         for line in self.config_toml.splitlines():
 293             if line.startswith(key + ' ='):
 294                 return self.get_string(line)
 295         return None
 296
 297     def get_mk(self, key):
 298         for line in iter(self.config_mk.splitlines()):
 299             if line.startswith(key + ' '):
 300                 var = line[line.find(':=') + 2:].strip()
 301                 if var != '':
 302                     return var
 303         return None
 304
 305     def cargo(self):
 306         config = self.get_toml('cargo')
 307         if config:
 308             return config
 309         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 310         if config:
 311             return config + '/bin/cargo' + self.exe_suffix()
 312         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 313
 314     def rustc(self):
 315         config = self.get_toml('rustc')
 316         if config:
 317             return config
 318         config = self.get_mk('CFG_LOCAL_RUST_ROOT')
 319         if config:
 320             return config + '/bin/rustc' + self.exe_suffix()
 321         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 322
 323     def get_string(self, line):
 324         start = line.find('"')
 325         end = start + 1 + line[start + 1:].find('"')
 326         return line[start + 1:end]
 327
 328     def exe_suffix(self):
 329         if sys.platform == 'win32':
 330             return '.exe'
 331         else:
 332             return ''
 333
 334     def print_what_it_means_to_bootstrap(self):
 335         if hasattr(self, 'printed'):
 336             return
 337         self.printed = True
 338         if os.path.exists(self.bootstrap_binary()):
 339             return
 340         if not '--help' in sys.argv or len(sys.argv) == 1:
 341             return
 342
 343         print('info: the build system for Rust is written in Rust, so this')
 344         print('      script is now going to download a stage0 rust compiler')
 345         print('      and then compile the build system itself')
 346         print('')
 347         print('info: in the meantime you can read more about rustbuild at')
 348         print('      src/bootstrap/README.md before the download finishes')
 349
 350     def bootstrap_binary(self):
 351         return os.path.join(self.build_dir, "bootstrap/debug/bootstrap")
 352
 353     def build_bootstrap(self):
 354         self.print_what_it_means_to_bootstrap()
 355         build_dir = os.path.join(self.build_dir, "bootstrap")
 356         if self.clean and os.path.exists(build_dir):
 357             shutil.rmtree(build_dir)
 358         env = os.environ.copy()
 359         env["CARGO_TARGET_DIR"] = build_dir
 360         env["RUSTC"] = self.rustc()
 361         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 362                                  (os.pathsep + env["LD_LIBRARY_PATH"]) \
 363                                  if "LD_LIBRARY_PATH" in env else ""
 364         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
 365                                    (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
 366                                    if "DYLD_LIBRARY_PATH" in env else ""
 367         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 368                       os.pathsep + env["PATH"]
 369         if not os.path.isfile(self.cargo()):
 370             raise Exception("no cargo executable found at `%s`" % self.cargo())
 371         args = [self.cargo(), "build", "--manifest-path",
 372                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
 373         if self.use_locked_deps:
 374             args.append("--locked")
 375         if self.use_vendored_sources:
 376             args.append("--frozen")
 377         self.run(args, env)
 378
 379     def run(self, args, env):
 380         proc = subprocess.Popen(args, env=env)
 381         ret = proc.wait()
 382         if ret != 0:
 383             sys.exit(ret)
 384
 385     def build_triple(self):
 386         default_encoding = sys.getdefaultencoding()
 387         config = self.get_toml('build')
 388         if config:
 389             return config
 390         config = self.get_mk('CFG_BUILD')
 391         if config:
 392             return config
 393         try:
 394             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 395             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 396         except (subprocess.CalledProcessError, OSError):
 397             if sys.platform == 'win32':
 398                 return 'x86_64-pc-windows-msvc'
 399             err = "uname not found"
 400             if self.verbose:
 401                 raise Exception(err)
 402             sys.exit(err)
 403
 404         # Darwin's `uname -s` lies and always returns i386. We have to use
 405         # sysctl instead.
 406         if ostype == 'Darwin' and cputype == 'i686':
 407             args = ['sysctl', 'hw.optional.x86_64']
 408             sysctl = subprocess.check_output(args).decode(default_encoding)
 409             if ': 1' in sysctl:
 410                 cputype = 'x86_64'
 411
 412         # The goal here is to come up with the same triple as LLVM would,
 413         # at least for the subset of platforms we're willing to target.
 414         if ostype == 'Linux':
 415             ostype = 'unknown-linux-gnu'
 416         elif ostype == 'FreeBSD':
 417             ostype = 'unknown-freebsd'
 418         elif ostype == 'DragonFly':
 419             ostype = 'unknown-dragonfly'
 420         elif ostype == 'Bitrig':
 421             ostype = 'unknown-bitrig'
 422         elif ostype == 'OpenBSD':
 423             ostype = 'unknown-openbsd'
 424         elif ostype == 'NetBSD':
 425             ostype = 'unknown-netbsd'
 426         elif ostype == 'SunOS':
 427             ostype = 'sun-solaris'
 428             # On Solaris, uname -m will return a machine classification instead
 429             # of a cpu type, so uname -p is recommended instead.  However, the
 430             # output from that option is too generic for our purposes (it will
 431             # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
 432             # must be used instead.
 433             try:
 434                 cputype = subprocess.check_output(['isainfo',
 435                   '-k']).strip().decode(default_encoding)
 436             except (subprocess.CalledProcessError, OSError):
 437                 err = "isainfo not found"
 438                 if self.verbose:
 439                     raise Exception(err)
 440                 sys.exit(err)
 441         elif ostype == 'Darwin':
 442             ostype = 'apple-darwin'
 443         elif ostype == 'Haiku':
 444             ostype = 'unknown-haiku'
 445         elif ostype.startswith('MINGW'):
 446             # msys' `uname` does not print gcc configuration, but prints msys
 447             # configuration. so we cannot believe `uname -m`:
 448             # msys1 is always i686 and msys2 is always x86_64.
 449             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 450             # MINGW64 on x86_64.
 451             ostype = 'pc-windows-gnu'
 452             cputype = 'i686'
 453             if os.environ.get('MSYSTEM') == 'MINGW64':
 454                 cputype = 'x86_64'
 455         elif ostype.startswith('MSYS'):
 456             ostype = 'pc-windows-gnu'
 457         elif ostype.startswith('CYGWIN_NT'):
 458             cputype = 'i686'
 459             if ostype.endswith('WOW64'):
 460                 cputype = 'x86_64'
 461             ostype = 'pc-windows-gnu'
 462         else:
 463             err = "unknown OS type: " + ostype
 464             if self.verbose:
 465                 raise ValueError(err)
 466             sys.exit(err)
 467
 468         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 469             cputype = 'i686'
 470         elif cputype in {'xscale', 'arm'}:
 471             cputype = 'arm'
 472         elif cputype in {'armv6l', 'armv7l', 'armv8l'}:
 473             cputype = 'arm'
 474             ostype += 'eabihf'
 475         elif cputype == 'armv7l':
 476             cputype = 'armv7'
 477             ostype += 'eabihf'
 478         elif cputype == 'aarch64':
 479             cputype = 'aarch64'
 480         elif cputype == 'arm64':
 481             cputype = 'aarch64'
 482         elif cputype == 'mips':
 483             if sys.byteorder == 'big':
 484                 cputype = 'mips'
 485             elif sys.byteorder == 'little':
 486                 cputype = 'mipsel'
 487             else:
 488                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 489         elif cputype == 'mips64':
 490             if sys.byteorder == 'big':
 491                 cputype = 'mips64'
 492             elif sys.byteorder == 'little':
 493                 cputype = 'mips64el'
 494             else:
 495                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 496             # only the n64 ABI is supported, indicate it
 497             ostype += 'abi64'
 498         elif cputype in {'powerpc', 'ppc'}:
 499             cputype = 'powerpc'
 500         elif cputype in {'powerpc64', 'ppc64'}:
 501             cputype = 'powerpc64'
 502         elif cputype in {'powerpc64le', 'ppc64le'}:
 503             cputype = 'powerpc64le'
 504         elif cputype == 'sparcv9':
 505             pass
 506         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 507             cputype = 'x86_64'
 508         elif cputype == 's390x':
 509             cputype = 's390x'
 510         elif cputype == 'BePC':
 511             cputype = 'i686'
 512         else:
 513             err = "unknown cpu type: " + cputype
 514             if self.verbose:
 515                 raise ValueError(err)
 516             sys.exit(err)
 517
 518         return "{}-{}".format(cputype, ostype)
 519
 520 def bootstrap():
 521     parser = argparse.ArgumentParser(description='Build rust')
 522     parser.add_argument('--config')
 523     parser.add_argument('--clean', action='store_true')
 524     parser.add_argument('-v', '--verbose', action='store_true')
 525
 526     args = [a for a in sys.argv if a != '-h' and a != '--help']
 527     args, _ = parser.parse_known_args(args)
 528
 529     # Configure initial bootstrap
 530     rb = RustBuild()
 531     rb.config_toml = ''
 532     rb.config_mk = ''
 533     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 534     rb.build_dir = os.path.join(os.getcwd(), "build")
 535     rb.verbose = args.verbose
 536     rb.clean = args.clean
 537
 538     try:
 539         with open(args.config or 'config.toml') as config:
 540             rb.config_toml = config.read()
 541     except:
 542         pass
 543     try:
 544         rb.config_mk = open('config.mk').read()
 545     except:
 546         pass
 547
 548     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
 549                               'CFG_ENABLE_VENDOR' in rb.config_mk
 550
 551     rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
 552                          'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
 553
 554     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
 555         if os.environ.get('USER') != os.environ['SUDO_USER']:
 556             rb.use_vendored_sources = True
 557             print('info: looks like you are running this command under `sudo`')
 558             print('      and so in order to preserve your $HOME this will now')
 559             print('      use vendored sources by default. Note that if this')
 560             print('      does not work you should run a normal build first')
 561             print('      before running a command like `sudo make install`')
 562
 563     if rb.use_vendored_sources:
 564         if not os.path.exists('.cargo'):
 565             os.makedirs('.cargo')
 566         with open('.cargo/config','w') as f:
 567             f.write("""
 568                 [source.crates-io]
 569                 replace-with = 'vendored-sources'
 570                 registry = 'https://example.com'
 571
 572                 [source.vendored-sources]
 573                 directory = '{}/src/vendor'
 574             """.format(rb.rust_root))
 575     else:
 576         if os.path.exists('.cargo'):
 577             shutil.rmtree('.cargo')
 578
 579     data = stage0_data(rb.rust_root)
 580     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 581     rb._cargo_rev = data['cargo']
 582
 583     # Fetch/build the bootstrap
 584     rb.build = rb.build_triple()
 585     rb.download_stage0()
 586     sys.stdout.flush()
 587     rb.build_bootstrap()
 588     sys.stdout.flush()
 589
 590     # Run the bootstrap
 591     args = [rb.bootstrap_binary()]
 592     args.extend(sys.argv[1:])
 593     env = os.environ.copy()
 594     env["BUILD"] = rb.build
 595     env["SRC"] = rb.rust_root
 596     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 597     rb.run(args, env)
 598
 599 def main():
 600     start_time = time()
 601     try:
 602         bootstrap()
 603         print("Build completed successfully in %s" % format_build_time(time() - start_time))
 604     except (SystemExit, KeyboardInterrupt) as e:
 605         if hasattr(e, 'code') and isinstance(e.code, int):
 606             exit_code = e.code
 607         else:
 608             exit_code = 1
 609             print(e)
 610         print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
 611         sys.exit(exit_code)
 612
 613 if __name__ == '__main__':
 614     main()