]>
Commit | Line | Data |
---|---|---|
abe05a73 | 1 | from __future__ import absolute_import, division, print_function |
7453a54e SL |
2 | import argparse |
3 | import contextlib | |
3157f602 | 4 | import datetime |
a7813a04 | 5 | import hashlib |
7453a54e | 6 | import os |
7cac9316 | 7 | import re |
7453a54e SL |
8 | import shutil |
9 | import subprocess | |
10 | import sys | |
11 | import tarfile | |
a7813a04 XL |
12 | import tempfile |
13 | ||
3157f602 XL |
14 | from time import time |
15 | ||
7453a54e SL |
16 | |
17 | def get(url, path, verbose=False): | |
041b39d2 XL |
18 | suffix = '.sha256' |
19 | sha_url = url + suffix | |
a7813a04 XL |
20 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: |
21 | temp_path = temp_file.name | |
041b39d2 | 22 | with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: |
a7813a04 XL |
23 | sha_path = sha_file.name |
24 | ||
25 | try: | |
476ff2be | 26 | download(sha_path, sha_url, False, verbose) |
5bcae85e SL |
27 | if os.path.exists(path): |
28 | if verify(path, sha_path, False): | |
476ff2be | 29 | if verbose: |
3b2f2976 | 30 | print("using already-download file", path) |
5bcae85e SL |
31 | return |
32 | else: | |
476ff2be | 33 | if verbose: |
3b2f2976 XL |
34 | print("ignoring already-download file", |
35 | path, "due to failed verification") | |
5bcae85e | 36 | os.unlink(path) |
476ff2be SL |
37 | download(temp_path, url, True, verbose) |
38 | if not verify(temp_path, sha_path, verbose): | |
5bcae85e | 39 | raise RuntimeError("failed verification") |
476ff2be SL |
40 | if verbose: |
41 | print("moving {} to {}".format(temp_path, path)) | |
a7813a04 XL |
42 | shutil.move(temp_path, path) |
43 | finally: | |
476ff2be SL |
44 | delete_if_present(sha_path, verbose) |
45 | delete_if_present(temp_path, verbose) | |
a7813a04 XL |
46 | |
47 | ||
476ff2be | 48 | def delete_if_present(path, verbose): |
041b39d2 | 49 | """Remove the given file if present""" |
a7813a04 | 50 | if os.path.isfile(path): |
476ff2be | 51 | if verbose: |
3b2f2976 | 52 | print("removing", path) |
a7813a04 XL |
53 | os.unlink(path) |
54 | ||
55 | ||
476ff2be | 56 | def download(path, url, probably_big, verbose): |
3b2f2976 | 57 | for _ in range(0, 4): |
8bb4bdeb XL |
58 | try: |
59 | _download(path, url, probably_big, verbose, True) | |
60 | return | |
61 | except RuntimeError: | |
62 | print("\nspurious failure, trying again") | |
63 | _download(path, url, probably_big, verbose, False) | |
64 | ||
65 | ||
66 | def _download(path, url, probably_big, verbose, exception): | |
476ff2be SL |
67 | if probably_big or verbose: |
68 | print("downloading {}".format(url)) | |
7453a54e SL |
69 | # see http://serverfault.com/questions/301128/how-to-download |
70 | if sys.platform == 'win32': | |
71 | run(["PowerShell.exe", "/nologo", "-Command", | |
a1dfa0c6 XL |
72 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", |
73 | "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], | |
8bb4bdeb XL |
74 | verbose=verbose, |
75 | exception=exception) | |
7453a54e | 76 | else: |
476ff2be SL |
77 | if probably_big or verbose: |
78 | option = "-#" | |
79 | else: | |
80 | option = "-s" | |
b7449926 XL |
81 | run(["curl", option, |
82 | "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds | |
83 | "--connect-timeout", "30", # timeout if cannot connect within 30 seconds | |
84 | "--retry", "3", "-Sf", "-o", path, url], | |
8bb4bdeb XL |
85 | verbose=verbose, |
86 | exception=exception) | |
7453a54e | 87 | |
a7813a04 XL |
88 | |
89 | def verify(path, sha_path, verbose): | |
041b39d2 | 90 | """Check if the sha256 sum of the given path is valid""" |
476ff2be | 91 | if verbose: |
3b2f2976 | 92 | print("verifying", path) |
041b39d2 XL |
93 | with open(path, "rb") as source: |
94 | found = hashlib.sha256(source.read()).hexdigest() | |
95 | with open(sha_path, "r") as sha256sum: | |
96 | expected = sha256sum.readline().split()[0] | |
5bcae85e | 97 | verified = found == expected |
476ff2be | 98 | if not verified: |
5bcae85e | 99 | print("invalid checksum:\n" |
7cac9316 XL |
100 | " found: {}\n" |
101 | " expected: {}".format(found, expected)) | |
5bcae85e | 102 | return verified |
a7813a04 XL |
103 | |
104 | ||
7453a54e | 105 | def unpack(tarball, dst, verbose=False, match=None): |
041b39d2 | 106 | """Unpack the given tarball file""" |
3b2f2976 | 107 | print("extracting", tarball) |
7453a54e SL |
108 | fname = os.path.basename(tarball).replace(".tar.gz", "") |
109 | with contextlib.closing(tarfile.open(tarball)) as tar: | |
3b2f2976 XL |
110 | for member in tar.getnames(): |
111 | if "/" not in member: | |
7453a54e | 112 | continue |
3b2f2976 | 113 | name = member.replace(fname + "/", "", 1) |
7453a54e SL |
114 | if match is not None and not name.startswith(match): |
115 | continue | |
116 | name = name[len(match) + 1:] | |
117 | ||
3b2f2976 | 118 | dst_path = os.path.join(dst, name) |
7453a54e | 119 | if verbose: |
3b2f2976 XL |
120 | print(" extracting", member) |
121 | tar.extract(member, dst) | |
122 | src_path = os.path.join(dst, member) | |
123 | if os.path.isdir(src_path) and os.path.exists(dst_path): | |
7453a54e | 124 | continue |
3b2f2976 | 125 | shutil.move(src_path, dst_path) |
7453a54e SL |
126 | shutil.rmtree(os.path.join(dst, fname)) |
127 | ||
041b39d2 | 128 | |
7cac9316 | 129 | def run(args, verbose=False, exception=False, **kwargs): |
3b2f2976 | 130 | """Run a child program in a new process""" |
7453a54e SL |
131 | if verbose: |
132 | print("running: " + ' '.join(args)) | |
133 | sys.stdout.flush() | |
134 | # Use Popen here instead of call() as it apparently allows powershell on | |
135 | # Windows to not lock up waiting for input presumably. | |
7cac9316 | 136 | ret = subprocess.Popen(args, **kwargs) |
7453a54e SL |
137 | code = ret.wait() |
138 | if code != 0: | |
a7813a04 | 139 | err = "failed to run: " + ' '.join(args) |
8bb4bdeb | 140 | if verbose or exception: |
a7813a04 XL |
141 | raise RuntimeError(err) |
142 | sys.exit(err) | |
143 | ||
7cac9316 | 144 | |
a7813a04 | 145 | def stage0_data(rust_root): |
3b2f2976 | 146 | """Build a dictionary from stage0.txt""" |
a7813a04 XL |
147 | nightlies = os.path.join(rust_root, "src/stage0.txt") |
148 | with open(nightlies, 'r') as nightlies: | |
3b2f2976 XL |
149 | lines = [line.rstrip() for line in nightlies |
150 | if not line.startswith("#")] | |
151 | return dict([line.split(": ", 1) for line in lines if line]) | |
3157f602 | 152 | |
7cac9316 | 153 | |
3157f602 | 154 | def format_build_time(duration): |
3b2f2976 XL |
155 | """Return a nicer format for build time |
156 | ||
157 | >>> format_build_time('300') | |
158 | '0:05:00' | |
159 | """ | |
3157f602 | 160 | return str(datetime.timedelta(seconds=int(duration))) |
7453a54e | 161 | |
9e0c209e | 162 | |
ea8adc8c XL |
163 | def default_build_triple(): |
164 | """Build triple as in LLVM""" | |
165 | default_encoding = sys.getdefaultencoding() | |
166 | try: | |
167 | ostype = subprocess.check_output( | |
168 | ['uname', '-s']).strip().decode(default_encoding) | |
169 | cputype = subprocess.check_output( | |
170 | ['uname', '-m']).strip().decode(default_encoding) | |
171 | except (subprocess.CalledProcessError, OSError): | |
172 | if sys.platform == 'win32': | |
173 | return 'x86_64-pc-windows-msvc' | |
174 | err = "uname not found" | |
175 | sys.exit(err) | |
176 | ||
177 | # The goal here is to come up with the same triple as LLVM would, | |
178 | # at least for the subset of platforms we're willing to target. | |
179 | ostype_mapper = { | |
ea8adc8c XL |
180 | 'Darwin': 'apple-darwin', |
181 | 'DragonFly': 'unknown-dragonfly', | |
182 | 'FreeBSD': 'unknown-freebsd', | |
183 | 'Haiku': 'unknown-haiku', | |
184 | 'NetBSD': 'unknown-netbsd', | |
185 | 'OpenBSD': 'unknown-openbsd' | |
186 | } | |
187 | ||
188 | # Consider the direct transformation first and then the special cases | |
189 | if ostype in ostype_mapper: | |
190 | ostype = ostype_mapper[ostype] | |
191 | elif ostype == 'Linux': | |
192 | os_from_sp = subprocess.check_output( | |
193 | ['uname', '-o']).strip().decode(default_encoding) | |
194 | if os_from_sp == 'Android': | |
195 | ostype = 'linux-android' | |
196 | else: | |
197 | ostype = 'unknown-linux-gnu' | |
198 | elif ostype == 'SunOS': | |
199 | ostype = 'sun-solaris' | |
200 | # On Solaris, uname -m will return a machine classification instead | |
201 | # of a cpu type, so uname -p is recommended instead. However, the | |
202 | # output from that option is too generic for our purposes (it will | |
203 | # always emit 'i386' on x86/amd64 systems). As such, isainfo -k | |
204 | # must be used instead. | |
205 | try: | |
206 | cputype = subprocess.check_output( | |
207 | ['isainfo', '-k']).strip().decode(default_encoding) | |
208 | except (subprocess.CalledProcessError, OSError): | |
209 | err = "isainfo not found" | |
210 | sys.exit(err) | |
211 | elif ostype.startswith('MINGW'): | |
212 | # msys' `uname` does not print gcc configuration, but prints msys | |
213 | # configuration. so we cannot believe `uname -m`: | |
214 | # msys1 is always i686 and msys2 is always x86_64. | |
215 | # instead, msys defines $MSYSTEM which is MINGW32 on i686 and | |
216 | # MINGW64 on x86_64. | |
217 | ostype = 'pc-windows-gnu' | |
218 | cputype = 'i686' | |
219 | if os.environ.get('MSYSTEM') == 'MINGW64': | |
220 | cputype = 'x86_64' | |
221 | elif ostype.startswith('MSYS'): | |
222 | ostype = 'pc-windows-gnu' | |
223 | elif ostype.startswith('CYGWIN_NT'): | |
224 | cputype = 'i686' | |
225 | if ostype.endswith('WOW64'): | |
226 | cputype = 'x86_64' | |
227 | ostype = 'pc-windows-gnu' | |
228 | else: | |
229 | err = "unknown OS type: {}".format(ostype) | |
230 | sys.exit(err) | |
231 | ||
9fa01778 XL |
232 | if cputype == 'powerpc' and ostype == 'unknown-freebsd': |
233 | cputype = subprocess.check_output( | |
234 | ['uname', '-p']).strip().decode(default_encoding) | |
ea8adc8c XL |
235 | cputype_mapper = { |
236 | 'BePC': 'i686', | |
237 | 'aarch64': 'aarch64', | |
238 | 'amd64': 'x86_64', | |
239 | 'arm64': 'aarch64', | |
240 | 'i386': 'i686', | |
241 | 'i486': 'i686', | |
242 | 'i686': 'i686', | |
243 | 'i786': 'i686', | |
244 | 'powerpc': 'powerpc', | |
245 | 'powerpc64': 'powerpc64', | |
246 | 'powerpc64le': 'powerpc64le', | |
247 | 'ppc': 'powerpc', | |
248 | 'ppc64': 'powerpc64', | |
249 | 'ppc64le': 'powerpc64le', | |
250 | 's390x': 's390x', | |
251 | 'x64': 'x86_64', | |
252 | 'x86': 'i686', | |
253 | 'x86-64': 'x86_64', | |
254 | 'x86_64': 'x86_64' | |
255 | } | |
256 | ||
257 | # Consider the direct transformation first and then the special cases | |
258 | if cputype in cputype_mapper: | |
259 | cputype = cputype_mapper[cputype] | |
260 | elif cputype in {'xscale', 'arm'}: | |
261 | cputype = 'arm' | |
262 | if ostype == 'linux-android': | |
263 | ostype = 'linux-androideabi' | |
532ac7d7 XL |
264 | elif ostype == 'unknown-freebsd': |
265 | cputype = subprocess.check_output( | |
266 | ['uname', '-p']).strip().decode(default_encoding) | |
267 | ostype = 'unknown-freebsd' | |
ea8adc8c XL |
268 | elif cputype == 'armv6l': |
269 | cputype = 'arm' | |
270 | if ostype == 'linux-android': | |
271 | ostype = 'linux-androideabi' | |
272 | else: | |
273 | ostype += 'eabihf' | |
274 | elif cputype in {'armv7l', 'armv8l'}: | |
275 | cputype = 'armv7' | |
276 | if ostype == 'linux-android': | |
277 | ostype = 'linux-androideabi' | |
278 | else: | |
279 | ostype += 'eabihf' | |
280 | elif cputype == 'mips': | |
281 | if sys.byteorder == 'big': | |
282 | cputype = 'mips' | |
283 | elif sys.byteorder == 'little': | |
284 | cputype = 'mipsel' | |
285 | else: | |
286 | raise ValueError("unknown byteorder: {}".format(sys.byteorder)) | |
287 | elif cputype == 'mips64': | |
288 | if sys.byteorder == 'big': | |
289 | cputype = 'mips64' | |
290 | elif sys.byteorder == 'little': | |
291 | cputype = 'mips64el' | |
292 | else: | |
293 | raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) | |
294 | # only the n64 ABI is supported, indicate it | |
295 | ostype += 'abi64' | |
0531ce1d | 296 | elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': |
ea8adc8c XL |
297 | pass |
298 | else: | |
299 | err = "unknown cpu type: {}".format(cputype) | |
300 | sys.exit(err) | |
301 | ||
302 | return "{}-{}".format(cputype, ostype) | |
303 | ||
abe05a73 | 304 | |
8faf50e0 XL |
305 | @contextlib.contextmanager |
306 | def output(filepath): | |
307 | tmp = filepath + '.tmp' | |
308 | with open(tmp, 'w') as f: | |
309 | yield f | |
310 | try: | |
311 | os.remove(filepath) # PermissionError/OSError on Win32 if in use | |
312 | os.rename(tmp, filepath) | |
313 | except OSError: | |
314 | shutil.copy2(tmp, filepath) | |
315 | os.remove(tmp) | |
316 | ||
317 | ||
9e0c209e | 318 | class RustBuild(object): |
3b2f2976 XL |
319 | """Provide all the methods required to build Rust""" |
320 | def __init__(self): | |
321 | self.cargo_channel = '' | |
322 | self.date = '' | |
e1599b0c | 323 | self._download_url = '' |
3b2f2976 XL |
324 | self.rustc_channel = '' |
325 | self.build = '' | |
326 | self.build_dir = os.path.join(os.getcwd(), "build") | |
327 | self.clean = False | |
3b2f2976 | 328 | self.config_toml = '' |
83c7162d | 329 | self.rust_root = '' |
3b2f2976 XL |
330 | self.use_locked_deps = '' |
331 | self.use_vendored_sources = '' | |
332 | self.verbose = False | |
7cac9316 | 333 | |
a7813a04 | 334 | def download_stage0(self): |
3b2f2976 XL |
335 | """Fetch the build system for Rust, written in Rust |
336 | ||
337 | This method will build a cache directory, then it will fetch the | |
338 | tarball which has the stage0 compiler used to then bootstrap the Rust | |
339 | compiler itself. | |
cc61c64b | 340 | |
3b2f2976 XL |
341 | Each downloaded tarball is extracted, after that, the script |
342 | will move all the content to the right place. | |
343 | """ | |
344 | rustc_channel = self.rustc_channel | |
345 | cargo_channel = self.cargo_channel | |
7453a54e SL |
346 | |
347 | if self.rustc().startswith(self.bin_root()) and \ | |
3b2f2976 XL |
348 | (not os.path.exists(self.rustc()) or |
349 | self.program_out_of_date(self.rustc_stamp())): | |
54a0048b SL |
350 | if os.path.exists(self.bin_root()): |
351 | shutil.rmtree(self.bin_root()) | |
7cac9316 XL |
352 | filename = "rust-std-{}-{}.tar.gz".format( |
353 | rustc_channel, self.build) | |
3b2f2976 XL |
354 | pattern = "rust-std-{}".format(self.build) |
355 | self._download_stage0_helper(filename, pattern) | |
7453a54e | 356 | |
7cac9316 | 357 | filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build) |
3b2f2976 XL |
358 | self._download_stage0_helper(filename, "rustc") |
359 | self.fix_executable("{}/bin/rustc".format(self.bin_root())) | |
360 | self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) | |
8faf50e0 | 361 | with output(self.rustc_stamp()) as rust_stamp: |
3b2f2976 | 362 | rust_stamp.write(self.date) |
7453a54e | 363 | |
0531ce1d XL |
364 | # This is required so that we don't mix incompatible MinGW |
365 | # libraries/binaries that are included in rust-std with | |
366 | # the system MinGW ones. | |
367 | if "pc-windows-gnu" in self.build: | |
368 | filename = "rust-mingw-{}-{}.tar.gz".format( | |
369 | rustc_channel, self.build) | |
370 | self._download_stage0_helper(filename, "rust-mingw") | |
371 | ||
7453a54e | 372 | if self.cargo().startswith(self.bin_root()) and \ |
3b2f2976 XL |
373 | (not os.path.exists(self.cargo()) or |
374 | self.program_out_of_date(self.cargo_stamp())): | |
7cac9316 | 375 | filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build) |
3b2f2976 XL |
376 | self._download_stage0_helper(filename, "cargo") |
377 | self.fix_executable("{}/bin/cargo".format(self.bin_root())) | |
8faf50e0 | 378 | with output(self.cargo_stamp()) as cargo_stamp: |
3b2f2976 XL |
379 | cargo_stamp.write(self.date) |
380 | ||
381 | def _download_stage0_helper(self, filename, pattern): | |
382 | cache_dst = os.path.join(self.build_dir, "cache") | |
383 | rustc_cache = os.path.join(cache_dst, self.date) | |
384 | if not os.path.exists(rustc_cache): | |
385 | os.makedirs(rustc_cache) | |
386 | ||
387 | url = "{}/dist/{}".format(self._download_url, self.date) | |
388 | tarball = os.path.join(rustc_cache, filename) | |
389 | if not os.path.exists(tarball): | |
390 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) | |
391 | unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose) | |
8bb4bdeb | 392 | |
3b2f2976 XL |
393 | @staticmethod |
394 | def fix_executable(fname): | |
395 | """Modifies the interpreter section of 'fname' to fix the dynamic linker | |
396 | ||
397 | This method is only required on NixOS and uses the PatchELF utility to | |
398 | change the dynamic linker of ELF executables. | |
399 | ||
400 | Please see https://nixos.org/patchelf.html for more information | |
401 | """ | |
8bb4bdeb XL |
402 | default_encoding = sys.getdefaultencoding() |
403 | try: | |
7cac9316 XL |
404 | ostype = subprocess.check_output( |
405 | ['uname', '-s']).strip().decode(default_encoding) | |
3b2f2976 | 406 | except subprocess.CalledProcessError: |
8bb4bdeb | 407 | return |
3b2f2976 XL |
408 | except OSError as reason: |
409 | if getattr(reason, 'winerror', None) is not None: | |
410 | return | |
411 | raise reason | |
8bb4bdeb XL |
412 | |
413 | if ostype != "Linux": | |
414 | return | |
415 | ||
416 | if not os.path.exists("/etc/NIXOS"): | |
417 | return | |
418 | if os.path.exists("/lib"): | |
419 | return | |
420 | ||
421 | # At this point we're pretty sure the user is running NixOS | |
041b39d2 XL |
422 | nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" |
423 | print(nix_os_msg, fname) | |
8bb4bdeb XL |
424 | |
425 | try: | |
7cac9316 XL |
426 | interpreter = subprocess.check_output( |
427 | ["patchelf", "--print-interpreter", fname]) | |
8bb4bdeb | 428 | interpreter = interpreter.strip().decode(default_encoding) |
3b2f2976 XL |
429 | except subprocess.CalledProcessError as reason: |
430 | print("warning: failed to call patchelf:", reason) | |
8bb4bdeb XL |
431 | return |
432 | ||
433 | loader = interpreter.split("/")[-1] | |
434 | ||
435 | try: | |
7cac9316 XL |
436 | ldd_output = subprocess.check_output( |
437 | ['ldd', '/run/current-system/sw/bin/sh']) | |
8bb4bdeb | 438 | ldd_output = ldd_output.strip().decode(default_encoding) |
3b2f2976 XL |
439 | except subprocess.CalledProcessError as reason: |
440 | print("warning: unable to call ldd:", reason) | |
8bb4bdeb XL |
441 | return |
442 | ||
443 | for line in ldd_output.splitlines(): | |
444 | libname = line.split()[0] | |
445 | if libname.endswith(loader): | |
446 | loader_path = libname[:len(libname) - len(loader)] | |
447 | break | |
448 | else: | |
449 | print("warning: unable to find the path to the dynamic linker") | |
450 | return | |
451 | ||
452 | correct_interpreter = loader_path + loader | |
453 | ||
454 | try: | |
7cac9316 XL |
455 | subprocess.check_output( |
456 | ["patchelf", "--set-interpreter", correct_interpreter, fname]) | |
3b2f2976 XL |
457 | except subprocess.CalledProcessError as reason: |
458 | print("warning: failed to call patchelf:", reason) | |
8bb4bdeb XL |
459 | return |
460 | ||
7453a54e | 461 | def rustc_stamp(self): |
3b2f2976 XL |
462 | """Return the path for .rustc-stamp |
463 | ||
464 | >>> rb = RustBuild() | |
465 | >>> rb.build_dir = "build" | |
466 | >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") | |
467 | True | |
468 | """ | |
7453a54e SL |
469 | return os.path.join(self.bin_root(), '.rustc-stamp') |
470 | ||
471 | def cargo_stamp(self): | |
3b2f2976 | 472 | """Return the path for .cargo-stamp |
7453a54e | 473 | |
3b2f2976 XL |
474 | >>> rb = RustBuild() |
475 | >>> rb.build_dir = "build" | |
476 | >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp") | |
477 | True | |
478 | """ | |
479 | return os.path.join(self.bin_root(), '.cargo-stamp') | |
7453a54e | 480 | |
3b2f2976 XL |
481 | def program_out_of_date(self, stamp_path): |
482 | """Check if the given program stamp is out of date""" | |
483 | if not os.path.exists(stamp_path) or self.clean: | |
7453a54e | 484 | return True |
3b2f2976 XL |
485 | with open(stamp_path, 'r') as stamp: |
486 | return self.date != stamp.read() | |
7453a54e SL |
487 | |
488 | def bin_root(self): | |
3b2f2976 XL |
489 | """Return the binary root directory |
490 | ||
491 | >>> rb = RustBuild() | |
492 | >>> rb.build_dir = "build" | |
493 | >>> rb.bin_root() == os.path.join("build", "stage0") | |
494 | True | |
495 | ||
496 | When the 'build' property is given should be a nested directory: | |
497 | ||
498 | >>> rb.build = "devel" | |
499 | >>> rb.bin_root() == os.path.join("build", "devel", "stage0") | |
500 | True | |
501 | """ | |
7453a54e SL |
502 | return os.path.join(self.build_dir, self.build, "stage0") |
503 | ||
94b46f34 | 504 | def get_toml(self, key, section=None): |
3b2f2976 XL |
505 | """Returns the value of the given key in config.toml, otherwise returns None |
506 | ||
507 | >>> rb = RustBuild() | |
508 | >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' | |
509 | >>> rb.get_toml("key2") | |
510 | 'value2' | |
511 | ||
512 | If the key does not exists, the result is None: | |
513 | ||
abe05a73 | 514 | >>> rb.get_toml("key3") is None |
3b2f2976 | 515 | True |
94b46f34 XL |
516 | |
517 | Optionally also matches the section the key appears in | |
518 | ||
519 | >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' | |
520 | >>> rb.get_toml('key', 'a') | |
521 | 'value1' | |
522 | >>> rb.get_toml('key', 'b') | |
523 | 'value2' | |
524 | >>> rb.get_toml('key', 'c') is None | |
525 | True | |
e1599b0c XL |
526 | |
527 | >>> rb.config_toml = 'key1 = true' | |
528 | >>> rb.get_toml("key1") | |
529 | 'true' | |
3b2f2976 | 530 | """ |
94b46f34 XL |
531 | |
532 | cur_section = None | |
7453a54e | 533 | for line in self.config_toml.splitlines(): |
94b46f34 XL |
534 | section_match = re.match(r'^\s*\[(.*)\]\s*$', line) |
535 | if section_match is not None: | |
536 | cur_section = section_match.group(1) | |
537 | ||
7cac9316 XL |
538 | match = re.match(r'^{}\s*=(.*)$'.format(key), line) |
539 | if match is not None: | |
540 | value = match.group(1) | |
94b46f34 XL |
541 | if section is None or section == cur_section: |
542 | return self.get_string(value) or value.strip() | |
7453a54e SL |
543 | return None |
544 | ||
7453a54e | 545 | def cargo(self): |
3b2f2976 XL |
546 | """Return config path for cargo""" |
547 | return self.program_config('cargo') | |
7453a54e SL |
548 | |
549 | def rustc(self): | |
3b2f2976 XL |
550 | """Return config path for rustc""" |
551 | return self.program_config('rustc') | |
552 | ||
553 | def program_config(self, program): | |
554 | """Return config path for the given program | |
555 | ||
556 | >>> rb = RustBuild() | |
557 | >>> rb.config_toml = 'rustc = "rustc"\\n' | |
3b2f2976 XL |
558 | >>> rb.program_config('rustc') |
559 | 'rustc' | |
3b2f2976 | 560 | >>> rb.config_toml = '' |
3b2f2976 XL |
561 | >>> cargo_path = rb.program_config('cargo') |
562 | >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), | |
563 | ... "bin", "cargo") | |
564 | True | |
565 | """ | |
566 | config = self.get_toml(program) | |
7453a54e | 567 | if config: |
abe05a73 | 568 | return os.path.expanduser(config) |
3b2f2976 XL |
569 | return os.path.join(self.bin_root(), "bin", "{}{}".format( |
570 | program, self.exe_suffix())) | |
571 | ||
572 | @staticmethod | |
573 | def get_string(line): | |
574 | """Return the value between double quotes | |
575 | ||
576 | >>> RustBuild.get_string(' "devel" ') | |
577 | 'devel' | |
e1599b0c XL |
578 | >>> RustBuild.get_string(" 'devel' ") |
579 | 'devel' | |
580 | >>> RustBuild.get_string('devel') is None | |
581 | True | |
582 | >>> RustBuild.get_string(' "devel ') | |
583 | '' | |
3b2f2976 | 584 | """ |
7453a54e | 585 | start = line.find('"') |
ea8adc8c XL |
586 | if start != -1: |
587 | end = start + 1 + line[start + 1:].find('"') | |
588 | return line[start + 1:end] | |
589 | start = line.find('\'') | |
590 | if start != -1: | |
591 | end = start + 1 + line[start + 1:].find('\'') | |
592 | return line[start + 1:end] | |
593 | return None | |
7453a54e | 594 | |
3b2f2976 XL |
595 | @staticmethod |
596 | def exe_suffix(): | |
597 | """Return a suffix for executables""" | |
7453a54e SL |
598 | if sys.platform == 'win32': |
599 | return '.exe' | |
041b39d2 | 600 | return '' |
7453a54e | 601 | |
476ff2be | 602 | def bootstrap_binary(self): |
0bf4aa26 | 603 | """Return the path of the bootstrap binary |
3b2f2976 XL |
604 | |
605 | >>> rb = RustBuild() | |
606 | >>> rb.build_dir = "build" | |
607 | >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", | |
608 | ... "debug", "bootstrap") | |
609 | True | |
610 | """ | |
611 | return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") | |
476ff2be | 612 | |
7453a54e | 613 | def build_bootstrap(self): |
3b2f2976 | 614 | """Build bootstrap""" |
3157f602 XL |
615 | build_dir = os.path.join(self.build_dir, "bootstrap") |
616 | if self.clean and os.path.exists(build_dir): | |
617 | shutil.rmtree(build_dir) | |
7453a54e | 618 | env = os.environ.copy() |
041b39d2 | 619 | env["RUSTC_BOOTSTRAP"] = '1' |
3157f602 | 620 | env["CARGO_TARGET_DIR"] = build_dir |
7453a54e | 621 | env["RUSTC"] = self.rustc() |
8bb4bdeb | 622 | env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ |
7cac9316 XL |
623 | (os.pathsep + env["LD_LIBRARY_PATH"]) \ |
624 | if "LD_LIBRARY_PATH" in env else "" | |
8bb4bdeb | 625 | env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ |
7cac9316 XL |
626 | (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ |
627 | if "DYLD_LIBRARY_PATH" in env else "" | |
628 | env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ | |
629 | (os.pathsep + env["LIBRARY_PATH"]) \ | |
630 | if "LIBRARY_PATH" in env else "" | |
94b46f34 XL |
631 | env["RUSTFLAGS"] = "-Cdebuginfo=2 " |
632 | ||
633 | build_section = "target.{}".format(self.build_triple()) | |
634 | target_features = [] | |
635 | if self.get_toml("crt-static", build_section) == "true": | |
636 | target_features += ["+crt-static"] | |
637 | elif self.get_toml("crt-static", build_section) == "false": | |
638 | target_features += ["-crt-static"] | |
639 | if target_features: | |
640 | env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " " | |
13cf67c4 XL |
641 | target_linker = self.get_toml("linker", build_section) |
642 | if target_linker is not None: | |
643 | env["RUSTFLAGS"] += "-C linker=" + target_linker + " " | |
e1599b0c XL |
644 | env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes " |
645 | if self.get_toml("deny-warnings", "rust") != "false": | |
646 | env["RUSTFLAGS"] += "-Dwarnings " | |
94b46f34 | 647 | |
7453a54e | 648 | env["PATH"] = os.path.join(self.bin_root(), "bin") + \ |
7cac9316 | 649 | os.pathsep + env["PATH"] |
32a655c1 | 650 | if not os.path.isfile(self.cargo()): |
3b2f2976 XL |
651 | raise Exception("no cargo executable found at `{}`".format( |
652 | self.cargo())) | |
476ff2be SL |
653 | args = [self.cargo(), "build", "--manifest-path", |
654 | os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] | |
0531ce1d | 655 | for _ in range(1, self.verbose): |
7cac9316 | 656 | args.append("--verbose") |
8bb4bdeb XL |
657 | if self.use_locked_deps: |
658 | args.append("--locked") | |
476ff2be SL |
659 | if self.use_vendored_sources: |
660 | args.append("--frozen") | |
7cac9316 | 661 | run(args, env=env, verbose=self.verbose) |
7453a54e SL |
662 | |
663 | def build_triple(self): | |
3b2f2976 | 664 | """Build triple as in LLVM""" |
7453a54e SL |
665 | config = self.get_toml('build') |
666 | if config: | |
667 | return config | |
ea8adc8c | 668 | return default_build_triple() |
7453a54e | 669 | |
0531ce1d XL |
670 | def check_submodule(self, module, slow_submodules): |
671 | if not slow_submodules: | |
672 | checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], | |
673 | cwd=os.path.join(self.rust_root, module), | |
674 | stdout=subprocess.PIPE) | |
675 | return checked_out | |
676 | else: | |
677 | return None | |
678 | ||
679 | def update_submodule(self, module, checked_out, recorded_submodules): | |
680 | module_path = os.path.join(self.rust_root, module) | |
681 | ||
e1599b0c | 682 | if checked_out is not None: |
0531ce1d XL |
683 | default_encoding = sys.getdefaultencoding() |
684 | checked_out = checked_out.communicate()[0].decode(default_encoding).strip() | |
685 | if recorded_submodules[module] == checked_out: | |
686 | return | |
687 | ||
688 | print("Updating submodule", module) | |
689 | ||
690 | run(["git", "submodule", "-q", "sync", module], | |
691 | cwd=self.rust_root, verbose=self.verbose) | |
48663c56 XL |
692 | try: |
693 | run(["git", "submodule", "update", | |
694 | "--init", "--recursive", "--progress", module], | |
695 | cwd=self.rust_root, verbose=self.verbose, exception=True) | |
696 | except RuntimeError: | |
697 | # Some versions of git don't support --progress. | |
698 | run(["git", "submodule", "update", | |
699 | "--init", "--recursive", module], | |
700 | cwd=self.rust_root, verbose=self.verbose) | |
0531ce1d XL |
701 | run(["git", "reset", "-q", "--hard"], |
702 | cwd=module_path, verbose=self.verbose) | |
703 | run(["git", "clean", "-qdfx"], | |
704 | cwd=module_path, verbose=self.verbose) | |
705 | ||
7cac9316 | 706 | def update_submodules(self): |
3b2f2976 | 707 | """Update submodules""" |
7cac9316 | 708 | if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ |
ea8adc8c | 709 | self.get_toml('submodules') == "false": |
7cac9316 | 710 | return |
e1599b0c XL |
711 | |
712 | # check the existence of 'git' command | |
713 | try: | |
714 | subprocess.check_output(['git', '--version']) | |
715 | except (subprocess.CalledProcessError, OSError): | |
716 | print("error: `git` is not found, please make sure it's installed and in the path.") | |
717 | sys.exit(1) | |
718 | ||
0531ce1d XL |
719 | slow_submodules = self.get_toml('fast-submodules') == "false" |
720 | start_time = time() | |
721 | if slow_submodules: | |
722 | print('Unconditionally updating all submodules') | |
723 | else: | |
724 | print('Updating only changed submodules') | |
7cac9316 | 725 | default_encoding = sys.getdefaultencoding() |
7cac9316 | 726 | submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( |
041b39d2 XL |
727 | ["git", "config", "--file", |
728 | os.path.join(self.rust_root, ".gitmodules"), | |
7cac9316 XL |
729 | "--get-regexp", "path"] |
730 | ).decode(default_encoding).splitlines()] | |
2c00a5a8 | 731 | filtered_submodules = [] |
0531ce1d | 732 | submodules_names = [] |
2c00a5a8 | 733 | for module in submodules: |
9fa01778 XL |
734 | if module.endswith("llvm-project"): |
735 | if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': | |
2c00a5a8 | 736 | continue |
0531ce1d XL |
737 | check = self.check_submodule(module, slow_submodules) |
738 | filtered_submodules.append((module, check)) | |
739 | submodules_names.append(module) | |
740 | recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, | |
741 | cwd=self.rust_root, stdout=subprocess.PIPE) | |
742 | recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() | |
743 | recorded_submodules = {} | |
744 | for data in recorded: | |
745 | data = data.split() | |
746 | recorded_submodules[data[3]] = data[2] | |
747 | for module in filtered_submodules: | |
748 | self.update_submodule(module[0], module[1], recorded_submodules) | |
749 | print("Submodules updated in %.2f seconds" % (time() - start_time)) | |
7cac9316 | 750 | |
e1599b0c XL |
751 | def set_normal_environment(self): |
752 | """Set download URL for normal environment""" | |
753 | if 'RUSTUP_DIST_SERVER' in os.environ: | |
754 | self._download_url = os.environ['RUSTUP_DIST_SERVER'] | |
755 | else: | |
756 | self._download_url = 'https://static.rust-lang.org' | |
757 | ||
3b2f2976 XL |
758 | def set_dev_environment(self): |
759 | """Set download URL for development environment""" | |
e1599b0c XL |
760 | if 'RUSTUP_DEV_DIST_SERVER' in os.environ: |
761 | self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] | |
762 | else: | |
763 | self._download_url = 'https://dev-static.rust-lang.org' | |
3b2f2976 | 764 | |
416331ca XL |
765 | def check_vendored_status(self): |
766 | """Check that vendoring is configured properly""" | |
767 | vendor_dir = os.path.join(self.rust_root, 'vendor') | |
768 | if 'SUDO_USER' in os.environ and not self.use_vendored_sources: | |
769 | if os.environ.get('USER') != os.environ['SUDO_USER']: | |
770 | self.use_vendored_sources = True | |
771 | print('info: looks like you are running this command under `sudo`') | |
772 | print(' and so in order to preserve your $HOME this will now') | |
773 | print(' use vendored sources by default.') | |
774 | if not os.path.exists(vendor_dir): | |
775 | print('error: vendoring required, but vendor directory does not exist.') | |
776 | print(' Run `cargo vendor` without sudo to initialize the ' | |
777 | 'vendor directory.') | |
778 | raise Exception("{} not found".format(vendor_dir)) | |
779 | ||
780 | if self.use_vendored_sources: | |
781 | if not os.path.exists('.cargo'): | |
782 | os.makedirs('.cargo') | |
783 | with output('.cargo/config') as cargo_config: | |
784 | cargo_config.write( | |
785 | "[source.crates-io]\n" | |
786 | "replace-with = 'vendored-sources'\n" | |
787 | "registry = 'https://example.com'\n" | |
788 | "\n" | |
789 | "[source.vendored-sources]\n" | |
790 | "directory = '{}/vendor'\n" | |
791 | .format(self.rust_root)) | |
792 | else: | |
793 | if os.path.exists('.cargo'): | |
794 | shutil.rmtree('.cargo') | |
795 | ||
796 | def ensure_vendored(self): | |
797 | """Ensure that the vendored sources are available if needed""" | |
798 | vendor_dir = os.path.join(self.rust_root, 'vendor') | |
799 | # Note that this does not handle updating the vendored dependencies if | |
800 | # the rust git repository is updated. Normal development usually does | |
801 | # not use vendoring, so hopefully this isn't too much of a problem. | |
802 | if self.use_vendored_sources and not os.path.exists(vendor_dir): | |
803 | run([self.cargo(), "vendor"], | |
804 | verbose=self.verbose, cwd=self.rust_root) | |
805 | ||
7cac9316 | 806 | |
0531ce1d | 807 | def bootstrap(help_triggered): |
3b2f2976 | 808 | """Configure, fetch, build and run the initial bootstrap""" |
0531ce1d XL |
809 | |
810 | # If the user is asking for help, let them know that the whole download-and-build | |
811 | # process has to happen before anything is printed out. | |
812 | if help_triggered: | |
813 | print("info: Downloading and building bootstrap before processing --help") | |
814 | print(" command. See src/bootstrap/README.md for help with common") | |
815 | print(" commands.") | |
816 | ||
a7813a04 XL |
817 | parser = argparse.ArgumentParser(description='Build rust') |
818 | parser.add_argument('--config') | |
3b2f2976 | 819 | parser.add_argument('--build') |
83c7162d | 820 | parser.add_argument('--src') |
3157f602 | 821 | parser.add_argument('--clean', action='store_true') |
0531ce1d | 822 | parser.add_argument('-v', '--verbose', action='count', default=0) |
a7813a04 | 823 | |
5bcae85e | 824 | args = [a for a in sys.argv if a != '-h' and a != '--help'] |
a7813a04 XL |
825 | args, _ = parser.parse_known_args(args) |
826 | ||
827 | # Configure initial bootstrap | |
3b2f2976 | 828 | build = RustBuild() |
83c7162d | 829 | build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) |
3b2f2976 XL |
830 | build.verbose = args.verbose |
831 | build.clean = args.clean | |
a7813a04 XL |
832 | |
833 | try: | |
834 | with open(args.config or 'config.toml') as config: | |
3b2f2976 | 835 | build.config_toml = config.read() |
ea8adc8c | 836 | except (OSError, IOError): |
a7813a04 XL |
837 | pass |
838 | ||
e1599b0c XL |
839 | config_verbose = build.get_toml('verbose', 'build') |
840 | if config_verbose is not None: | |
841 | build.verbose = max(build.verbose, int(config_verbose)) | |
7cac9316 | 842 | |
e1599b0c | 843 | build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' |
476ff2be | 844 | |
e1599b0c | 845 | build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' |
8bb4bdeb | 846 | |
416331ca | 847 | build.check_vendored_status() |
476ff2be | 848 | |
3b2f2976 XL |
849 | data = stage0_data(build.rust_root) |
850 | build.date = data['date'] | |
851 | build.rustc_channel = data['rustc'] | |
852 | build.cargo_channel = data['cargo'] | |
853 | ||
7cac9316 | 854 | if 'dev' in data: |
3b2f2976 | 855 | build.set_dev_environment() |
e1599b0c XL |
856 | else: |
857 | build.set_normal_environment() | |
7cac9316 | 858 | |
83c7162d | 859 | build.update_submodules() |
a7813a04 XL |
860 | |
861 | # Fetch/build the bootstrap | |
3b2f2976 XL |
862 | build.build = args.build or build.build_triple() |
863 | build.download_stage0() | |
a7813a04 | 864 | sys.stdout.flush() |
416331ca | 865 | build.ensure_vendored() |
3b2f2976 | 866 | build.build_bootstrap() |
a7813a04 XL |
867 | sys.stdout.flush() |
868 | ||
869 | # Run the bootstrap | |
3b2f2976 | 870 | args = [build.bootstrap_binary()] |
a7813a04 XL |
871 | args.extend(sys.argv[1:]) |
872 | env = os.environ.copy() | |
3b2f2976 XL |
873 | env["BUILD"] = build.build |
874 | env["SRC"] = build.rust_root | |
a7813a04 | 875 | env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) |
041b39d2 | 876 | env["BOOTSTRAP_PYTHON"] = sys.executable |
83c7162d | 877 | env["BUILD_DIR"] = build.build_dir |
8faf50e0 XL |
878 | env["RUSTC_BOOTSTRAP"] = '1' |
879 | env["CARGO"] = build.cargo() | |
880 | env["RUSTC"] = build.rustc() | |
3b2f2976 | 881 | run(args, env=env, verbose=build.verbose) |
7cac9316 | 882 | |
a7813a04 | 883 | |
8bb4bdeb | 884 | def main(): |
3b2f2976 | 885 | """Entry point for the bootstrap process""" |
8bb4bdeb | 886 | start_time = time() |
0bf4aa26 XL |
887 | |
888 | # x.py help <cmd> ... | |
889 | if len(sys.argv) > 1 and sys.argv[1] == 'help': | |
532ac7d7 | 890 | sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] |
0bf4aa26 | 891 | |
7cac9316 XL |
892 | help_triggered = ( |
893 | '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) | |
8bb4bdeb | 894 | try: |
0531ce1d | 895 | bootstrap(help_triggered) |
cc61c64b | 896 | if not help_triggered: |
3b2f2976 XL |
897 | print("Build completed successfully in {}".format( |
898 | format_build_time(time() - start_time))) | |
899 | except (SystemExit, KeyboardInterrupt) as error: | |
900 | if hasattr(error, 'code') and isinstance(error.code, int): | |
901 | exit_code = error.code | |
8bb4bdeb XL |
902 | else: |
903 | exit_code = 1 | |
3b2f2976 | 904 | print(error) |
cc61c64b | 905 | if not help_triggered: |
3b2f2976 XL |
906 | print("Build completed unsuccessfully in {}".format( |
907 | format_build_time(time() - start_time))) | |
8bb4bdeb | 908 | sys.exit(exit_code) |
3157f602 | 909 | |
041b39d2 | 910 | |
a7813a04 XL |
911 | if __name__ == '__main__': |
912 | main() |