src/bootstrap/bootstrap.py

   1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
   2 # file at the top-level directory of this distribution and at
   3 # http://rust-lang.org/COPYRIGHT.
   4 #
   5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 # option. This file may not be copied, modified, or distributed
   9 # except according to those terms.
  10
  11 import argparse
  12 import contextlib
  13 import datetime
  14 import hashlib
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tarfile
  20 import tempfile
  21
  22 from time import time
  23
  24
  25 def get(url, path, verbose=False):
  26     sha_url = url + ".sha256"
  27     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
  28         temp_path = temp_file.name
  29     with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
  30         sha_path = sha_file.name
  31
  32     try:
  33         download(sha_path, sha_url, verbose)
  34         if os.path.exists(path):
  35             if verify(path, sha_path, False):
  36                 print("using already-download file " + path)
  37                 return
  38             else:
  39                 print("ignoring already-download file " + path + " due to failed verification")
  40                 os.unlink(path)
  41         download(temp_path, url, verbose)
  42         if not verify(temp_path, sha_path, True):
  43             raise RuntimeError("failed verification")
  44         print("moving {} to {}".format(temp_path, path))
  45         shutil.move(temp_path, path)
  46     finally:
  47         delete_if_present(sha_path)
  48         delete_if_present(temp_path)
  49
  50
  51 def delete_if_present(path):
  52     if os.path.isfile(path):
  53         print("removing " + path)
  54         os.unlink(path)
  55
  56
  57 def download(path, url, verbose):
  58     print("downloading {} to {}".format(url, path))
  59     # see http://serverfault.com/questions/301128/how-to-download
  60     if sys.platform == 'win32':
  61         run(["PowerShell.exe", "/nologo", "-Command",
  62              "(New-Object System.Net.WebClient)"
  63              ".DownloadFile('{}', '{}')".format(url, path)],
  64             verbose=verbose)
  65     else:
  66         run(["curl", "-o", path, url], verbose=verbose)
  67
  68
  69 def verify(path, sha_path, verbose):
  70     print("verifying " + path)
  71     with open(path, "rb") as f:
  72         found = hashlib.sha256(f.read()).hexdigest()
  73     with open(sha_path, "r") as f:
  74         expected, _ = f.readline().split()
  75     verified = found == expected
  76     if not verified and verbose:
  77         print("invalid checksum:\n"
  78                "    found:    {}\n"
  79                "    expected: {}".format(found, expected))
  80     return verified
  81
  82
  83 def unpack(tarball, dst, verbose=False, match=None):
  84     print("extracting " + tarball)
  85     fname = os.path.basename(tarball).replace(".tar.gz", "")
  86     with contextlib.closing(tarfile.open(tarball)) as tar:
  87         for p in tar.getnames():
  88             if "/" not in p:
  89                 continue
  90             name = p.replace(fname + "/", "", 1)
  91             if match is not None and not name.startswith(match):
  92                 continue
  93             name = name[len(match) + 1:]
  94
  95             fp = os.path.join(dst, name)
  96             if verbose:
  97                 print("  extracting " + p)
  98             tar.extract(p, dst)
  99             tp = os.path.join(dst, p)
 100             if os.path.isdir(tp) and os.path.exists(fp):
 101                 continue
 102             shutil.move(tp, fp)
 103     shutil.rmtree(os.path.join(dst, fname))
 104
 105 def run(args, verbose=False):
 106     if verbose:
 107         print("running: " + ' '.join(args))
 108     sys.stdout.flush()
 109     # Use Popen here instead of call() as it apparently allows powershell on
 110     # Windows to not lock up waiting for input presumably.
 111     ret = subprocess.Popen(args)
 112     code = ret.wait()
 113     if code != 0:
 114         err = "failed to run: " + ' '.join(args)
 115         if verbose:
 116             raise RuntimeError(err)
 117         sys.exit(err)
 118
 119 def stage0_data(rust_root):
 120     nightlies = os.path.join(rust_root, "src/stage0.txt")
 121     data = {}
 122     with open(nightlies, 'r') as nightlies:
 123         for line in nightlies:
 124             line = line.rstrip()  # Strip newline character, '\n'
 125             if line.startswith("#") or line == '':
 126                 continue
 127             a, b = line.split(": ", 1)
 128             data[a] = b
 129     return data
 130
 131 def format_build_time(duration):
 132     return str(datetime.timedelta(seconds=int(duration)))
 133
 134
 135 class RustBuild(object):
 136     def download_stage0(self):
 137         cache_dst = os.path.join(self.build_dir, "cache")
 138         rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
 139         cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
 140         if not os.path.exists(rustc_cache):
 141             os.makedirs(rustc_cache)
 142         if not os.path.exists(cargo_cache):
 143             os.makedirs(cargo_cache)
 144
 145         if self.rustc().startswith(self.bin_root()) and \
 146                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
 147             if os.path.exists(self.bin_root()):
 148                 shutil.rmtree(self.bin_root())
 149             channel = self.stage0_rustc_channel()
 150             filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
 151             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 152             tarball = os.path.join(rustc_cache, filename)
 153             if not os.path.exists(tarball):
 154                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 155             unpack(tarball, self.bin_root(),
 156                    match="rust-std-" + self.build,
 157                    verbose=self.verbose)
 158
 159             filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
 160             url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
 161             tarball = os.path.join(rustc_cache, filename)
 162             if not os.path.exists(tarball):
 163                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 164             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
 165             with open(self.rustc_stamp(), 'w') as f:
 166                 f.write(self.stage0_rustc_date())
 167
 168         if self.cargo().startswith(self.bin_root()) and \
 169                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
 170             channel = self.stage0_cargo_channel()
 171             filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
 172             url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
 173             tarball = os.path.join(cargo_cache, filename)
 174             if not os.path.exists(tarball):
 175                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
 176             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
 177             with open(self.cargo_stamp(), 'w') as f:
 178                 f.write(self.stage0_cargo_date())
 179
 180     def stage0_cargo_date(self):
 181         return self._cargo_date
 182
 183     def stage0_cargo_channel(self):
 184         return self._cargo_channel
 185
 186     def stage0_rustc_date(self):
 187         return self._rustc_date
 188
 189     def stage0_rustc_channel(self):
 190         return self._rustc_channel
 191
 192     def rustc_stamp(self):
 193         return os.path.join(self.bin_root(), '.rustc-stamp')
 194
 195     def cargo_stamp(self):
 196         return os.path.join(self.bin_root(), '.cargo-stamp')
 197
 198     def rustc_out_of_date(self):
 199         if not os.path.exists(self.rustc_stamp()) or self.clean:
 200             return True
 201         with open(self.rustc_stamp(), 'r') as f:
 202             return self.stage0_rustc_date() != f.read()
 203
 204     def cargo_out_of_date(self):
 205         if not os.path.exists(self.cargo_stamp()) or self.clean:
 206             return True
 207         with open(self.cargo_stamp(), 'r') as f:
 208             return self.stage0_cargo_date() != f.read()
 209
 210     def bin_root(self):
 211         return os.path.join(self.build_dir, self.build, "stage0")
 212
 213     def get_toml(self, key):
 214         for line in self.config_toml.splitlines():
 215             if line.startswith(key + ' ='):
 216                 return self.get_string(line)
 217         return None
 218
 219     def get_mk(self, key):
 220         for line in iter(self.config_mk.splitlines()):
 221             if line.startswith(key):
 222                 return line[line.find(':=') + 2:].strip()
 223         return None
 224
 225     def cargo(self):
 226         config = self.get_toml('cargo')
 227         if config:
 228             return config
 229         return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix())
 230
 231     def rustc(self):
 232         config = self.get_toml('rustc')
 233         if config:
 234             return config
 235         config = self.get_mk('CFG_LOCAL_RUST')
 236         if config:
 237             return config + '/bin/rustc' + self.exe_suffix()
 238         return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix())
 239
 240     def get_string(self, line):
 241         start = line.find('"')
 242         end = start + 1 + line[start + 1:].find('"')
 243         return line[start + 1:end]
 244
 245     def exe_suffix(self):
 246         if sys.platform == 'win32':
 247             return '.exe'
 248         else:
 249             return ''
 250
 251     def build_bootstrap(self):
 252         build_dir = os.path.join(self.build_dir, "bootstrap")
 253         if self.clean and os.path.exists(build_dir):
 254             shutil.rmtree(build_dir)
 255         env = os.environ.copy()
 256         env["CARGO_TARGET_DIR"] = build_dir
 257         env["RUSTC"] = self.rustc()
 258         env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 259         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
 260         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
 261                       os.pathsep + env["PATH"]
 262         self.run([self.cargo(), "build", "--manifest-path",
 263                   os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
 264                  env)
 265
 266     def run(self, args, env):
 267         proc = subprocess.Popen(args, env=env)
 268         ret = proc.wait()
 269         if ret != 0:
 270             sys.exit(ret)
 271
 272     def build_triple(self):
 273         default_encoding = sys.getdefaultencoding()
 274         config = self.get_toml('build')
 275         if config:
 276             return config
 277         config = self.get_mk('CFG_BUILD')
 278         if config:
 279             return config
 280         try:
 281             ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
 282             cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
 283         except (subprocess.CalledProcessError, WindowsError):
 284             if sys.platform == 'win32':
 285                 return 'x86_64-pc-windows-msvc'
 286             err = "uname not found"
 287             if self.verbose:
 288                 raise Exception(err)
 289             sys.exit(err)
 290
 291         # Darwin's `uname -s` lies and always returns i386. We have to use
 292         # sysctl instead.
 293         if ostype == 'Darwin' and cputype == 'i686':
 294             args = ['sysctl', 'hw.optional.x86_64']
 295             sysctl = subprocess.check_output(args).decode(default_encoding)
 296             if ': 1' in sysctl:
 297                 cputype = 'x86_64'
 298
 299         # The goal here is to come up with the same triple as LLVM would,
 300         # at least for the subset of platforms we're willing to target.
 301         if ostype == 'Linux':
 302             ostype = 'unknown-linux-gnu'
 303         elif ostype == 'FreeBSD':
 304             ostype = 'unknown-freebsd'
 305         elif ostype == 'DragonFly':
 306             ostype = 'unknown-dragonfly'
 307         elif ostype == 'Bitrig':
 308             ostype = 'unknown-bitrig'
 309         elif ostype == 'OpenBSD':
 310             ostype = 'unknown-openbsd'
 311         elif ostype == 'NetBSD':
 312             ostype = 'unknown-netbsd'
 313         elif ostype == 'Darwin':
 314             ostype = 'apple-darwin'
 315         elif ostype.startswith('MINGW'):
 316             # msys' `uname` does not print gcc configuration, but prints msys
 317             # configuration. so we cannot believe `uname -m`:
 318             # msys1 is always i686 and msys2 is always x86_64.
 319             # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
 320             # MINGW64 on x86_64.
 321             ostype = 'pc-windows-gnu'
 322             cputype = 'i686'
 323             if os.environ.get('MSYSTEM') == 'MINGW64':
 324                 cputype = 'x86_64'
 325         elif ostype.startswith('MSYS'):
 326             ostype = 'pc-windows-gnu'
 327         elif ostype.startswith('CYGWIN_NT'):
 328             cputype = 'i686'
 329             if ostype.endswith('WOW64'):
 330                 cputype = 'x86_64'
 331             ostype = 'pc-windows-gnu'
 332         else:
 333             err = "unknown OS type: " + ostype
 334             if self.verbose:
 335                 raise ValueError(err)
 336             sys.exit(err)
 337
 338         if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}:
 339             cputype = 'i686'
 340         elif cputype in {'xscale', 'arm'}:
 341             cputype = 'arm'
 342         elif cputype == 'armv7l':
 343             cputype = 'arm'
 344             ostype += 'eabihf'
 345         elif cputype == 'aarch64':
 346             cputype = 'aarch64'
 347         elif cputype == 'mips':
 348             if sys.byteorder == 'big':
 349                 cputype = 'mips'
 350             elif sys.byteorder == 'little':
 351                 cputype = 'mipsel'
 352             else:
 353                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 354         elif cputype == 'mips64':
 355             if sys.byteorder == 'big':
 356                 cputype = 'mips64'
 357             elif sys.byteorder == 'little':
 358                 cputype = 'mips64el'
 359             else:
 360                 raise ValueError('unknown byteorder: ' + sys.byteorder)
 361             # only the n64 ABI is supported, indicate it
 362             ostype += 'abi64'
 363         elif cputype in {'powerpc', 'ppc', 'ppc64'}:
 364             cputype = 'powerpc'
 365         elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}:
 366             cputype = 'x86_64'
 367         else:
 368             err = "unknown cpu type: " + cputype
 369             if self.verbose:
 370                 raise ValueError(err)
 371             sys.exit(err)
 372
 373         return "{}-{}".format(cputype, ostype)
 374
 375 def main():
 376     parser = argparse.ArgumentParser(description='Build rust')
 377     parser.add_argument('--config')
 378     parser.add_argument('--clean', action='store_true')
 379     parser.add_argument('-v', '--verbose', action='store_true')
 380
 381     args = [a for a in sys.argv if a != '-h' and a != '--help']
 382     args, _ = parser.parse_known_args(args)
 383
 384     # Configure initial bootstrap
 385     rb = RustBuild()
 386     rb.config_toml = ''
 387     rb.config_mk = ''
 388     rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
 389     rb.build_dir = os.path.join(os.getcwd(), "build")
 390     rb.verbose = args.verbose
 391     rb.clean = args.clean
 392
 393     try:
 394         with open(args.config or 'config.toml') as config:
 395             rb.config_toml = config.read()
 396     except:
 397         pass
 398     try:
 399         rb.config_mk = open('config.mk').read()
 400     except:
 401         pass
 402
 403     data = stage0_data(rb.rust_root)
 404     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
 405     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
 406
 407     start_time = time()
 408
 409     # Fetch/build the bootstrap
 410     rb.build = rb.build_triple()
 411     rb.download_stage0()
 412     sys.stdout.flush()
 413     rb.build_bootstrap()
 414     sys.stdout.flush()
 415
 416     # Run the bootstrap
 417     args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
 418     args.extend(sys.argv[1:])
 419     env = os.environ.copy()
 420     env["BUILD"] = rb.build
 421     env["SRC"] = rb.rust_root
 422     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
 423     rb.run(args, env)
 424
 425     end_time = time()
 426
 427     print("Build completed in %s" % format_build_time(end_time - start_time))
 428
 429 if __name__ == '__main__':
 430     main()