]>
Commit | Line | Data |
---|---|---|
abe05a73 | 1 | from __future__ import absolute_import, division, print_function |
7453a54e SL |
2 | import argparse |
3 | import contextlib | |
3157f602 | 4 | import datetime |
a7813a04 | 5 | import hashlib |
7453a54e | 6 | import os |
7cac9316 | 7 | import re |
7453a54e SL |
8 | import shutil |
9 | import subprocess | |
10 | import sys | |
11 | import tarfile | |
a7813a04 XL |
12 | import tempfile |
13 | ||
3157f602 XL |
14 | from time import time |
15 | ||
7453a54e SL |
16 | |
17 | def get(url, path, verbose=False): | |
041b39d2 XL |
18 | suffix = '.sha256' |
19 | sha_url = url + suffix | |
a7813a04 XL |
20 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: |
21 | temp_path = temp_file.name | |
041b39d2 | 22 | with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: |
a7813a04 XL |
23 | sha_path = sha_file.name |
24 | ||
25 | try: | |
476ff2be | 26 | download(sha_path, sha_url, False, verbose) |
5bcae85e SL |
27 | if os.path.exists(path): |
28 | if verify(path, sha_path, False): | |
476ff2be | 29 | if verbose: |
3b2f2976 | 30 | print("using already-download file", path) |
5bcae85e SL |
31 | return |
32 | else: | |
476ff2be | 33 | if verbose: |
3b2f2976 XL |
34 | print("ignoring already-download file", |
35 | path, "due to failed verification") | |
5bcae85e | 36 | os.unlink(path) |
476ff2be SL |
37 | download(temp_path, url, True, verbose) |
38 | if not verify(temp_path, sha_path, verbose): | |
5bcae85e | 39 | raise RuntimeError("failed verification") |
476ff2be SL |
40 | if verbose: |
41 | print("moving {} to {}".format(temp_path, path)) | |
a7813a04 XL |
42 | shutil.move(temp_path, path) |
43 | finally: | |
476ff2be SL |
44 | delete_if_present(sha_path, verbose) |
45 | delete_if_present(temp_path, verbose) | |
a7813a04 XL |
46 | |
47 | ||
476ff2be | 48 | def delete_if_present(path, verbose): |
041b39d2 | 49 | """Remove the given file if present""" |
a7813a04 | 50 | if os.path.isfile(path): |
476ff2be | 51 | if verbose: |
3b2f2976 | 52 | print("removing", path) |
a7813a04 XL |
53 | os.unlink(path) |
54 | ||
55 | ||
476ff2be | 56 | def download(path, url, probably_big, verbose): |
3b2f2976 | 57 | for _ in range(0, 4): |
8bb4bdeb XL |
58 | try: |
59 | _download(path, url, probably_big, verbose, True) | |
60 | return | |
61 | except RuntimeError: | |
62 | print("\nspurious failure, trying again") | |
63 | _download(path, url, probably_big, verbose, False) | |
64 | ||
65 | ||
66 | def _download(path, url, probably_big, verbose, exception): | |
476ff2be SL |
67 | if probably_big or verbose: |
68 | print("downloading {}".format(url)) | |
7453a54e SL |
69 | # see http://serverfault.com/questions/301128/how-to-download |
70 | if sys.platform == 'win32': | |
71 | run(["PowerShell.exe", "/nologo", "-Command", | |
a1dfa0c6 XL |
72 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", |
73 | "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], | |
8bb4bdeb XL |
74 | verbose=verbose, |
75 | exception=exception) | |
7453a54e | 76 | else: |
476ff2be SL |
77 | if probably_big or verbose: |
78 | option = "-#" | |
79 | else: | |
80 | option = "-s" | |
b7449926 XL |
81 | run(["curl", option, |
82 | "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds | |
83 | "--connect-timeout", "30", # timeout if cannot connect within 30 seconds | |
84 | "--retry", "3", "-Sf", "-o", path, url], | |
8bb4bdeb XL |
85 | verbose=verbose, |
86 | exception=exception) | |
7453a54e | 87 | |
a7813a04 XL |
88 | |
89 | def verify(path, sha_path, verbose): | |
041b39d2 | 90 | """Check if the sha256 sum of the given path is valid""" |
476ff2be | 91 | if verbose: |
3b2f2976 | 92 | print("verifying", path) |
041b39d2 XL |
93 | with open(path, "rb") as source: |
94 | found = hashlib.sha256(source.read()).hexdigest() | |
95 | with open(sha_path, "r") as sha256sum: | |
96 | expected = sha256sum.readline().split()[0] | |
5bcae85e | 97 | verified = found == expected |
476ff2be | 98 | if not verified: |
5bcae85e | 99 | print("invalid checksum:\n" |
7cac9316 XL |
100 | " found: {}\n" |
101 | " expected: {}".format(found, expected)) | |
5bcae85e | 102 | return verified |
a7813a04 XL |
103 | |
104 | ||
7453a54e | 105 | def unpack(tarball, dst, verbose=False, match=None): |
041b39d2 | 106 | """Unpack the given tarball file""" |
3b2f2976 | 107 | print("extracting", tarball) |
7453a54e SL |
108 | fname = os.path.basename(tarball).replace(".tar.gz", "") |
109 | with contextlib.closing(tarfile.open(tarball)) as tar: | |
3b2f2976 XL |
110 | for member in tar.getnames(): |
111 | if "/" not in member: | |
7453a54e | 112 | continue |
3b2f2976 | 113 | name = member.replace(fname + "/", "", 1) |
7453a54e SL |
114 | if match is not None and not name.startswith(match): |
115 | continue | |
116 | name = name[len(match) + 1:] | |
117 | ||
3b2f2976 | 118 | dst_path = os.path.join(dst, name) |
7453a54e | 119 | if verbose: |
3b2f2976 XL |
120 | print(" extracting", member) |
121 | tar.extract(member, dst) | |
122 | src_path = os.path.join(dst, member) | |
123 | if os.path.isdir(src_path) and os.path.exists(dst_path): | |
7453a54e | 124 | continue |
3b2f2976 | 125 | shutil.move(src_path, dst_path) |
7453a54e SL |
126 | shutil.rmtree(os.path.join(dst, fname)) |
127 | ||
041b39d2 | 128 | |
7cac9316 | 129 | def run(args, verbose=False, exception=False, **kwargs): |
3b2f2976 | 130 | """Run a child program in a new process""" |
7453a54e SL |
131 | if verbose: |
132 | print("running: " + ' '.join(args)) | |
133 | sys.stdout.flush() | |
134 | # Use Popen here instead of call() as it apparently allows powershell on | |
135 | # Windows to not lock up waiting for input presumably. | |
7cac9316 | 136 | ret = subprocess.Popen(args, **kwargs) |
7453a54e SL |
137 | code = ret.wait() |
138 | if code != 0: | |
a7813a04 | 139 | err = "failed to run: " + ' '.join(args) |
8bb4bdeb | 140 | if verbose or exception: |
a7813a04 XL |
141 | raise RuntimeError(err) |
142 | sys.exit(err) | |
143 | ||
7cac9316 | 144 | |
a7813a04 | 145 | def stage0_data(rust_root): |
3b2f2976 | 146 | """Build a dictionary from stage0.txt""" |
a7813a04 XL |
147 | nightlies = os.path.join(rust_root, "src/stage0.txt") |
148 | with open(nightlies, 'r') as nightlies: | |
3b2f2976 XL |
149 | lines = [line.rstrip() for line in nightlies |
150 | if not line.startswith("#")] | |
151 | return dict([line.split(": ", 1) for line in lines if line]) | |
3157f602 | 152 | |
7cac9316 | 153 | |
3157f602 | 154 | def format_build_time(duration): |
3b2f2976 XL |
155 | """Return a nicer format for build time |
156 | ||
157 | >>> format_build_time('300') | |
158 | '0:05:00' | |
159 | """ | |
3157f602 | 160 | return str(datetime.timedelta(seconds=int(duration))) |
7453a54e | 161 | |
9e0c209e | 162 | |
ea8adc8c XL |
163 | def default_build_triple(): |
164 | """Build triple as in LLVM""" | |
165 | default_encoding = sys.getdefaultencoding() | |
166 | try: | |
167 | ostype = subprocess.check_output( | |
168 | ['uname', '-s']).strip().decode(default_encoding) | |
169 | cputype = subprocess.check_output( | |
170 | ['uname', '-m']).strip().decode(default_encoding) | |
171 | except (subprocess.CalledProcessError, OSError): | |
172 | if sys.platform == 'win32': | |
173 | return 'x86_64-pc-windows-msvc' | |
174 | err = "uname not found" | |
175 | sys.exit(err) | |
176 | ||
177 | # The goal here is to come up with the same triple as LLVM would, | |
178 | # at least for the subset of platforms we're willing to target. | |
179 | ostype_mapper = { | |
180 | 'Bitrig': 'unknown-bitrig', | |
181 | 'Darwin': 'apple-darwin', | |
182 | 'DragonFly': 'unknown-dragonfly', | |
183 | 'FreeBSD': 'unknown-freebsd', | |
184 | 'Haiku': 'unknown-haiku', | |
185 | 'NetBSD': 'unknown-netbsd', | |
186 | 'OpenBSD': 'unknown-openbsd' | |
187 | } | |
188 | ||
189 | # Consider the direct transformation first and then the special cases | |
190 | if ostype in ostype_mapper: | |
191 | ostype = ostype_mapper[ostype] | |
192 | elif ostype == 'Linux': | |
193 | os_from_sp = subprocess.check_output( | |
194 | ['uname', '-o']).strip().decode(default_encoding) | |
195 | if os_from_sp == 'Android': | |
196 | ostype = 'linux-android' | |
197 | else: | |
198 | ostype = 'unknown-linux-gnu' | |
199 | elif ostype == 'SunOS': | |
200 | ostype = 'sun-solaris' | |
201 | # On Solaris, uname -m will return a machine classification instead | |
202 | # of a cpu type, so uname -p is recommended instead. However, the | |
203 | # output from that option is too generic for our purposes (it will | |
204 | # always emit 'i386' on x86/amd64 systems). As such, isainfo -k | |
205 | # must be used instead. | |
206 | try: | |
207 | cputype = subprocess.check_output( | |
208 | ['isainfo', '-k']).strip().decode(default_encoding) | |
209 | except (subprocess.CalledProcessError, OSError): | |
210 | err = "isainfo not found" | |
211 | sys.exit(err) | |
212 | elif ostype.startswith('MINGW'): | |
213 | # msys' `uname` does not print gcc configuration, but prints msys | |
214 | # configuration. so we cannot believe `uname -m`: | |
215 | # msys1 is always i686 and msys2 is always x86_64. | |
216 | # instead, msys defines $MSYSTEM which is MINGW32 on i686 and | |
217 | # MINGW64 on x86_64. | |
218 | ostype = 'pc-windows-gnu' | |
219 | cputype = 'i686' | |
220 | if os.environ.get('MSYSTEM') == 'MINGW64': | |
221 | cputype = 'x86_64' | |
222 | elif ostype.startswith('MSYS'): | |
223 | ostype = 'pc-windows-gnu' | |
224 | elif ostype.startswith('CYGWIN_NT'): | |
225 | cputype = 'i686' | |
226 | if ostype.endswith('WOW64'): | |
227 | cputype = 'x86_64' | |
228 | ostype = 'pc-windows-gnu' | |
229 | else: | |
230 | err = "unknown OS type: {}".format(ostype) | |
231 | sys.exit(err) | |
232 | ||
9fa01778 XL |
233 | if cputype == 'powerpc' and ostype == 'unknown-freebsd': |
234 | cputype = subprocess.check_output( | |
235 | ['uname', '-p']).strip().decode(default_encoding) | |
ea8adc8c XL |
236 | cputype_mapper = { |
237 | 'BePC': 'i686', | |
238 | 'aarch64': 'aarch64', | |
239 | 'amd64': 'x86_64', | |
240 | 'arm64': 'aarch64', | |
241 | 'i386': 'i686', | |
242 | 'i486': 'i686', | |
243 | 'i686': 'i686', | |
244 | 'i786': 'i686', | |
245 | 'powerpc': 'powerpc', | |
246 | 'powerpc64': 'powerpc64', | |
247 | 'powerpc64le': 'powerpc64le', | |
248 | 'ppc': 'powerpc', | |
249 | 'ppc64': 'powerpc64', | |
250 | 'ppc64le': 'powerpc64le', | |
251 | 's390x': 's390x', | |
252 | 'x64': 'x86_64', | |
253 | 'x86': 'i686', | |
254 | 'x86-64': 'x86_64', | |
255 | 'x86_64': 'x86_64' | |
256 | } | |
257 | ||
258 | # Consider the direct transformation first and then the special cases | |
259 | if cputype in cputype_mapper: | |
260 | cputype = cputype_mapper[cputype] | |
261 | elif cputype in {'xscale', 'arm'}: | |
262 | cputype = 'arm' | |
263 | if ostype == 'linux-android': | |
264 | ostype = 'linux-androideabi' | |
265 | elif cputype == 'armv6l': | |
266 | cputype = 'arm' | |
267 | if ostype == 'linux-android': | |
268 | ostype = 'linux-androideabi' | |
269 | else: | |
270 | ostype += 'eabihf' | |
271 | elif cputype in {'armv7l', 'armv8l'}: | |
272 | cputype = 'armv7' | |
273 | if ostype == 'linux-android': | |
274 | ostype = 'linux-androideabi' | |
275 | else: | |
276 | ostype += 'eabihf' | |
277 | elif cputype == 'mips': | |
278 | if sys.byteorder == 'big': | |
279 | cputype = 'mips' | |
280 | elif sys.byteorder == 'little': | |
281 | cputype = 'mipsel' | |
282 | else: | |
283 | raise ValueError("unknown byteorder: {}".format(sys.byteorder)) | |
284 | elif cputype == 'mips64': | |
285 | if sys.byteorder == 'big': | |
286 | cputype = 'mips64' | |
287 | elif sys.byteorder == 'little': | |
288 | cputype = 'mips64el' | |
289 | else: | |
290 | raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) | |
291 | # only the n64 ABI is supported, indicate it | |
292 | ostype += 'abi64' | |
0531ce1d | 293 | elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': |
ea8adc8c XL |
294 | pass |
295 | else: | |
296 | err = "unknown cpu type: {}".format(cputype) | |
297 | sys.exit(err) | |
298 | ||
299 | return "{}-{}".format(cputype, ostype) | |
300 | ||
abe05a73 | 301 | |
8faf50e0 XL |
302 | @contextlib.contextmanager |
303 | def output(filepath): | |
304 | tmp = filepath + '.tmp' | |
305 | with open(tmp, 'w') as f: | |
306 | yield f | |
307 | try: | |
308 | os.remove(filepath) # PermissionError/OSError on Win32 if in use | |
309 | os.rename(tmp, filepath) | |
310 | except OSError: | |
311 | shutil.copy2(tmp, filepath) | |
312 | os.remove(tmp) | |
313 | ||
314 | ||
9e0c209e | 315 | class RustBuild(object): |
3b2f2976 XL |
316 | """Provide all the methods required to build Rust""" |
317 | def __init__(self): | |
318 | self.cargo_channel = '' | |
319 | self.date = '' | |
320 | self._download_url = 'https://static.rust-lang.org' | |
321 | self.rustc_channel = '' | |
322 | self.build = '' | |
323 | self.build_dir = os.path.join(os.getcwd(), "build") | |
324 | self.clean = False | |
3b2f2976 | 325 | self.config_toml = '' |
83c7162d | 326 | self.rust_root = '' |
3b2f2976 XL |
327 | self.use_locked_deps = '' |
328 | self.use_vendored_sources = '' | |
329 | self.verbose = False | |
7cac9316 | 330 | |
a7813a04 | 331 | def download_stage0(self): |
3b2f2976 XL |
332 | """Fetch the build system for Rust, written in Rust |
333 | ||
334 | This method will build a cache directory, then it will fetch the | |
335 | tarball which has the stage0 compiler used to then bootstrap the Rust | |
336 | compiler itself. | |
cc61c64b | 337 | |
3b2f2976 XL |
338 | Each downloaded tarball is extracted, after that, the script |
339 | will move all the content to the right place. | |
340 | """ | |
341 | rustc_channel = self.rustc_channel | |
342 | cargo_channel = self.cargo_channel | |
7453a54e SL |
343 | |
344 | if self.rustc().startswith(self.bin_root()) and \ | |
3b2f2976 XL |
345 | (not os.path.exists(self.rustc()) or |
346 | self.program_out_of_date(self.rustc_stamp())): | |
54a0048b SL |
347 | if os.path.exists(self.bin_root()): |
348 | shutil.rmtree(self.bin_root()) | |
7cac9316 XL |
349 | filename = "rust-std-{}-{}.tar.gz".format( |
350 | rustc_channel, self.build) | |
3b2f2976 XL |
351 | pattern = "rust-std-{}".format(self.build) |
352 | self._download_stage0_helper(filename, pattern) | |
7453a54e | 353 | |
7cac9316 | 354 | filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build) |
3b2f2976 XL |
355 | self._download_stage0_helper(filename, "rustc") |
356 | self.fix_executable("{}/bin/rustc".format(self.bin_root())) | |
357 | self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) | |
8faf50e0 | 358 | with output(self.rustc_stamp()) as rust_stamp: |
3b2f2976 | 359 | rust_stamp.write(self.date) |
7453a54e | 360 | |
0531ce1d XL |
361 | # This is required so that we don't mix incompatible MinGW |
362 | # libraries/binaries that are included in rust-std with | |
363 | # the system MinGW ones. | |
364 | if "pc-windows-gnu" in self.build: | |
365 | filename = "rust-mingw-{}-{}.tar.gz".format( | |
366 | rustc_channel, self.build) | |
367 | self._download_stage0_helper(filename, "rust-mingw") | |
368 | ||
7453a54e | 369 | if self.cargo().startswith(self.bin_root()) and \ |
3b2f2976 XL |
370 | (not os.path.exists(self.cargo()) or |
371 | self.program_out_of_date(self.cargo_stamp())): | |
7cac9316 | 372 | filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build) |
3b2f2976 XL |
373 | self._download_stage0_helper(filename, "cargo") |
374 | self.fix_executable("{}/bin/cargo".format(self.bin_root())) | |
8faf50e0 | 375 | with output(self.cargo_stamp()) as cargo_stamp: |
3b2f2976 XL |
376 | cargo_stamp.write(self.date) |
377 | ||
378 | def _download_stage0_helper(self, filename, pattern): | |
379 | cache_dst = os.path.join(self.build_dir, "cache") | |
380 | rustc_cache = os.path.join(cache_dst, self.date) | |
381 | if not os.path.exists(rustc_cache): | |
382 | os.makedirs(rustc_cache) | |
383 | ||
384 | url = "{}/dist/{}".format(self._download_url, self.date) | |
385 | tarball = os.path.join(rustc_cache, filename) | |
386 | if not os.path.exists(tarball): | |
387 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) | |
388 | unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose) | |
8bb4bdeb | 389 | |
3b2f2976 XL |
390 | @staticmethod |
391 | def fix_executable(fname): | |
392 | """Modifies the interpreter section of 'fname' to fix the dynamic linker | |
393 | ||
394 | This method is only required on NixOS and uses the PatchELF utility to | |
395 | change the dynamic linker of ELF executables. | |
396 | ||
397 | Please see https://nixos.org/patchelf.html for more information | |
398 | """ | |
8bb4bdeb XL |
399 | default_encoding = sys.getdefaultencoding() |
400 | try: | |
7cac9316 XL |
401 | ostype = subprocess.check_output( |
402 | ['uname', '-s']).strip().decode(default_encoding) | |
3b2f2976 | 403 | except subprocess.CalledProcessError: |
8bb4bdeb | 404 | return |
3b2f2976 XL |
405 | except OSError as reason: |
406 | if getattr(reason, 'winerror', None) is not None: | |
407 | return | |
408 | raise reason | |
8bb4bdeb XL |
409 | |
410 | if ostype != "Linux": | |
411 | return | |
412 | ||
413 | if not os.path.exists("/etc/NIXOS"): | |
414 | return | |
415 | if os.path.exists("/lib"): | |
416 | return | |
417 | ||
418 | # At this point we're pretty sure the user is running NixOS | |
041b39d2 XL |
419 | nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" |
420 | print(nix_os_msg, fname) | |
8bb4bdeb XL |
421 | |
422 | try: | |
7cac9316 XL |
423 | interpreter = subprocess.check_output( |
424 | ["patchelf", "--print-interpreter", fname]) | |
8bb4bdeb | 425 | interpreter = interpreter.strip().decode(default_encoding) |
3b2f2976 XL |
426 | except subprocess.CalledProcessError as reason: |
427 | print("warning: failed to call patchelf:", reason) | |
8bb4bdeb XL |
428 | return |
429 | ||
430 | loader = interpreter.split("/")[-1] | |
431 | ||
432 | try: | |
7cac9316 XL |
433 | ldd_output = subprocess.check_output( |
434 | ['ldd', '/run/current-system/sw/bin/sh']) | |
8bb4bdeb | 435 | ldd_output = ldd_output.strip().decode(default_encoding) |
3b2f2976 XL |
436 | except subprocess.CalledProcessError as reason: |
437 | print("warning: unable to call ldd:", reason) | |
8bb4bdeb XL |
438 | return |
439 | ||
440 | for line in ldd_output.splitlines(): | |
441 | libname = line.split()[0] | |
442 | if libname.endswith(loader): | |
443 | loader_path = libname[:len(libname) - len(loader)] | |
444 | break | |
445 | else: | |
446 | print("warning: unable to find the path to the dynamic linker") | |
447 | return | |
448 | ||
449 | correct_interpreter = loader_path + loader | |
450 | ||
451 | try: | |
7cac9316 XL |
452 | subprocess.check_output( |
453 | ["patchelf", "--set-interpreter", correct_interpreter, fname]) | |
3b2f2976 XL |
454 | except subprocess.CalledProcessError as reason: |
455 | print("warning: failed to call patchelf:", reason) | |
8bb4bdeb XL |
456 | return |
457 | ||
7453a54e | 458 | def rustc_stamp(self): |
3b2f2976 XL |
459 | """Return the path for .rustc-stamp |
460 | ||
461 | >>> rb = RustBuild() | |
462 | >>> rb.build_dir = "build" | |
463 | >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") | |
464 | True | |
465 | """ | |
7453a54e SL |
466 | return os.path.join(self.bin_root(), '.rustc-stamp') |
467 | ||
468 | def cargo_stamp(self): | |
3b2f2976 | 469 | """Return the path for .cargo-stamp |
7453a54e | 470 | |
3b2f2976 XL |
471 | >>> rb = RustBuild() |
472 | >>> rb.build_dir = "build" | |
473 | >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp") | |
474 | True | |
475 | """ | |
476 | return os.path.join(self.bin_root(), '.cargo-stamp') | |
7453a54e | 477 | |
3b2f2976 XL |
478 | def program_out_of_date(self, stamp_path): |
479 | """Check if the given program stamp is out of date""" | |
480 | if not os.path.exists(stamp_path) or self.clean: | |
7453a54e | 481 | return True |
3b2f2976 XL |
482 | with open(stamp_path, 'r') as stamp: |
483 | return self.date != stamp.read() | |
7453a54e SL |
484 | |
485 | def bin_root(self): | |
3b2f2976 XL |
486 | """Return the binary root directory |
487 | ||
488 | >>> rb = RustBuild() | |
489 | >>> rb.build_dir = "build" | |
490 | >>> rb.bin_root() == os.path.join("build", "stage0") | |
491 | True | |
492 | ||
493 | When the 'build' property is given should be a nested directory: | |
494 | ||
495 | >>> rb.build = "devel" | |
496 | >>> rb.bin_root() == os.path.join("build", "devel", "stage0") | |
497 | True | |
498 | """ | |
7453a54e SL |
499 | return os.path.join(self.build_dir, self.build, "stage0") |
500 | ||
94b46f34 | 501 | def get_toml(self, key, section=None): |
3b2f2976 XL |
502 | """Returns the value of the given key in config.toml, otherwise returns None |
503 | ||
504 | >>> rb = RustBuild() | |
505 | >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' | |
506 | >>> rb.get_toml("key2") | |
507 | 'value2' | |
508 | ||
509 | If the key does not exists, the result is None: | |
510 | ||
abe05a73 | 511 | >>> rb.get_toml("key3") is None |
3b2f2976 | 512 | True |
94b46f34 XL |
513 | |
514 | Optionally also matches the section the key appears in | |
515 | ||
516 | >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' | |
517 | >>> rb.get_toml('key', 'a') | |
518 | 'value1' | |
519 | >>> rb.get_toml('key', 'b') | |
520 | 'value2' | |
521 | >>> rb.get_toml('key', 'c') is None | |
522 | True | |
3b2f2976 | 523 | """ |
94b46f34 XL |
524 | |
525 | cur_section = None | |
7453a54e | 526 | for line in self.config_toml.splitlines(): |
94b46f34 XL |
527 | section_match = re.match(r'^\s*\[(.*)\]\s*$', line) |
528 | if section_match is not None: | |
529 | cur_section = section_match.group(1) | |
530 | ||
7cac9316 XL |
531 | match = re.match(r'^{}\s*=(.*)$'.format(key), line) |
532 | if match is not None: | |
533 | value = match.group(1) | |
94b46f34 XL |
534 | if section is None or section == cur_section: |
535 | return self.get_string(value) or value.strip() | |
7453a54e SL |
536 | return None |
537 | ||
7453a54e | 538 | def cargo(self): |
3b2f2976 XL |
539 | """Return config path for cargo""" |
540 | return self.program_config('cargo') | |
7453a54e SL |
541 | |
542 | def rustc(self): | |
3b2f2976 XL |
543 | """Return config path for rustc""" |
544 | return self.program_config('rustc') | |
545 | ||
546 | def program_config(self, program): | |
547 | """Return config path for the given program | |
548 | ||
549 | >>> rb = RustBuild() | |
550 | >>> rb.config_toml = 'rustc = "rustc"\\n' | |
3b2f2976 XL |
551 | >>> rb.program_config('rustc') |
552 | 'rustc' | |
3b2f2976 | 553 | >>> rb.config_toml = '' |
3b2f2976 XL |
554 | >>> cargo_path = rb.program_config('cargo') |
555 | >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), | |
556 | ... "bin", "cargo") | |
557 | True | |
558 | """ | |
559 | config = self.get_toml(program) | |
7453a54e | 560 | if config: |
abe05a73 | 561 | return os.path.expanduser(config) |
3b2f2976 XL |
562 | return os.path.join(self.bin_root(), "bin", "{}{}".format( |
563 | program, self.exe_suffix())) | |
564 | ||
565 | @staticmethod | |
566 | def get_string(line): | |
567 | """Return the value between double quotes | |
568 | ||
569 | >>> RustBuild.get_string(' "devel" ') | |
570 | 'devel' | |
571 | """ | |
7453a54e | 572 | start = line.find('"') |
ea8adc8c XL |
573 | if start != -1: |
574 | end = start + 1 + line[start + 1:].find('"') | |
575 | return line[start + 1:end] | |
576 | start = line.find('\'') | |
577 | if start != -1: | |
578 | end = start + 1 + line[start + 1:].find('\'') | |
579 | return line[start + 1:end] | |
580 | return None | |
7453a54e | 581 | |
3b2f2976 XL |
582 | @staticmethod |
583 | def exe_suffix(): | |
584 | """Return a suffix for executables""" | |
7453a54e SL |
585 | if sys.platform == 'win32': |
586 | return '.exe' | |
041b39d2 | 587 | return '' |
7453a54e | 588 | |
476ff2be | 589 | def bootstrap_binary(self): |
0bf4aa26 | 590 | """Return the path of the bootstrap binary |
3b2f2976 XL |
591 | |
592 | >>> rb = RustBuild() | |
593 | >>> rb.build_dir = "build" | |
594 | >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", | |
595 | ... "debug", "bootstrap") | |
596 | True | |
597 | """ | |
598 | return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") | |
476ff2be | 599 | |
7453a54e | 600 | def build_bootstrap(self): |
3b2f2976 | 601 | """Build bootstrap""" |
3157f602 XL |
602 | build_dir = os.path.join(self.build_dir, "bootstrap") |
603 | if self.clean and os.path.exists(build_dir): | |
604 | shutil.rmtree(build_dir) | |
7453a54e | 605 | env = os.environ.copy() |
041b39d2 | 606 | env["RUSTC_BOOTSTRAP"] = '1' |
3157f602 | 607 | env["CARGO_TARGET_DIR"] = build_dir |
7453a54e | 608 | env["RUSTC"] = self.rustc() |
8bb4bdeb | 609 | env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ |
7cac9316 XL |
610 | (os.pathsep + env["LD_LIBRARY_PATH"]) \ |
611 | if "LD_LIBRARY_PATH" in env else "" | |
8bb4bdeb | 612 | env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ |
7cac9316 XL |
613 | (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ |
614 | if "DYLD_LIBRARY_PATH" in env else "" | |
615 | env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ | |
616 | (os.pathsep + env["LIBRARY_PATH"]) \ | |
617 | if "LIBRARY_PATH" in env else "" | |
94b46f34 XL |
618 | env["RUSTFLAGS"] = "-Cdebuginfo=2 " |
619 | ||
620 | build_section = "target.{}".format(self.build_triple()) | |
621 | target_features = [] | |
622 | if self.get_toml("crt-static", build_section) == "true": | |
623 | target_features += ["+crt-static"] | |
624 | elif self.get_toml("crt-static", build_section) == "false": | |
625 | target_features += ["-crt-static"] | |
626 | if target_features: | |
627 | env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " " | |
13cf67c4 XL |
628 | target_linker = self.get_toml("linker", build_section) |
629 | if target_linker is not None: | |
630 | env["RUSTFLAGS"] += "-C linker=" + target_linker + " " | |
94b46f34 | 631 | |
7453a54e | 632 | env["PATH"] = os.path.join(self.bin_root(), "bin") + \ |
7cac9316 | 633 | os.pathsep + env["PATH"] |
32a655c1 | 634 | if not os.path.isfile(self.cargo()): |
3b2f2976 XL |
635 | raise Exception("no cargo executable found at `{}`".format( |
636 | self.cargo())) | |
476ff2be SL |
637 | args = [self.cargo(), "build", "--manifest-path", |
638 | os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] | |
0531ce1d | 639 | for _ in range(1, self.verbose): |
7cac9316 | 640 | args.append("--verbose") |
8bb4bdeb XL |
641 | if self.use_locked_deps: |
642 | args.append("--locked") | |
476ff2be SL |
643 | if self.use_vendored_sources: |
644 | args.append("--frozen") | |
7cac9316 | 645 | run(args, env=env, verbose=self.verbose) |
7453a54e SL |
646 | |
647 | def build_triple(self): | |
3b2f2976 | 648 | """Build triple as in LLVM""" |
7453a54e SL |
649 | config = self.get_toml('build') |
650 | if config: | |
651 | return config | |
ea8adc8c | 652 | return default_build_triple() |
7453a54e | 653 | |
0531ce1d XL |
654 | def check_submodule(self, module, slow_submodules): |
655 | if not slow_submodules: | |
656 | checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], | |
657 | cwd=os.path.join(self.rust_root, module), | |
658 | stdout=subprocess.PIPE) | |
659 | return checked_out | |
660 | else: | |
661 | return None | |
662 | ||
663 | def update_submodule(self, module, checked_out, recorded_submodules): | |
664 | module_path = os.path.join(self.rust_root, module) | |
665 | ||
666 | if checked_out != None: | |
667 | default_encoding = sys.getdefaultencoding() | |
668 | checked_out = checked_out.communicate()[0].decode(default_encoding).strip() | |
669 | if recorded_submodules[module] == checked_out: | |
670 | return | |
671 | ||
672 | print("Updating submodule", module) | |
673 | ||
674 | run(["git", "submodule", "-q", "sync", module], | |
675 | cwd=self.rust_root, verbose=self.verbose) | |
676 | run(["git", "submodule", "update", | |
0731742a | 677 | "--init", "--recursive", "--progress", module], |
0531ce1d XL |
678 | cwd=self.rust_root, verbose=self.verbose) |
679 | run(["git", "reset", "-q", "--hard"], | |
680 | cwd=module_path, verbose=self.verbose) | |
681 | run(["git", "clean", "-qdfx"], | |
682 | cwd=module_path, verbose=self.verbose) | |
683 | ||
7cac9316 | 684 | def update_submodules(self): |
3b2f2976 | 685 | """Update submodules""" |
7cac9316 | 686 | if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ |
ea8adc8c | 687 | self.get_toml('submodules') == "false": |
7cac9316 | 688 | return |
0531ce1d XL |
689 | slow_submodules = self.get_toml('fast-submodules') == "false" |
690 | start_time = time() | |
691 | if slow_submodules: | |
692 | print('Unconditionally updating all submodules') | |
693 | else: | |
694 | print('Updating only changed submodules') | |
7cac9316 | 695 | default_encoding = sys.getdefaultencoding() |
7cac9316 | 696 | submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( |
041b39d2 XL |
697 | ["git", "config", "--file", |
698 | os.path.join(self.rust_root, ".gitmodules"), | |
7cac9316 XL |
699 | "--get-regexp", "path"] |
700 | ).decode(default_encoding).splitlines()] | |
2c00a5a8 | 701 | filtered_submodules = [] |
0531ce1d | 702 | submodules_names = [] |
2c00a5a8 | 703 | for module in submodules: |
9fa01778 XL |
704 | if module.endswith("llvm-project"): |
705 | if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': | |
2c00a5a8 XL |
706 | continue |
707 | if module.endswith("llvm-emscripten"): | |
708 | backends = self.get_toml('codegen-backends') | |
709 | if backends is None or not 'emscripten' in backends: | |
710 | continue | |
0531ce1d XL |
711 | check = self.check_submodule(module, slow_submodules) |
712 | filtered_submodules.append((module, check)) | |
713 | submodules_names.append(module) | |
714 | recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, | |
715 | cwd=self.rust_root, stdout=subprocess.PIPE) | |
716 | recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() | |
717 | recorded_submodules = {} | |
718 | for data in recorded: | |
719 | data = data.split() | |
720 | recorded_submodules[data[3]] = data[2] | |
721 | for module in filtered_submodules: | |
722 | self.update_submodule(module[0], module[1], recorded_submodules) | |
723 | print("Submodules updated in %.2f seconds" % (time() - start_time)) | |
7cac9316 | 724 | |
3b2f2976 XL |
725 | def set_dev_environment(self): |
726 | """Set download URL for development environment""" | |
727 | self._download_url = 'https://dev-static.rust-lang.org' | |
728 | ||
7cac9316 | 729 | |
0531ce1d | 730 | def bootstrap(help_triggered): |
3b2f2976 | 731 | """Configure, fetch, build and run the initial bootstrap""" |
0531ce1d XL |
732 | |
733 | # If the user is asking for help, let them know that the whole download-and-build | |
734 | # process has to happen before anything is printed out. | |
735 | if help_triggered: | |
736 | print("info: Downloading and building bootstrap before processing --help") | |
737 | print(" command. See src/bootstrap/README.md for help with common") | |
738 | print(" commands.") | |
739 | ||
a7813a04 XL |
740 | parser = argparse.ArgumentParser(description='Build rust') |
741 | parser.add_argument('--config') | |
3b2f2976 | 742 | parser.add_argument('--build') |
83c7162d | 743 | parser.add_argument('--src') |
3157f602 | 744 | parser.add_argument('--clean', action='store_true') |
0531ce1d | 745 | parser.add_argument('-v', '--verbose', action='count', default=0) |
a7813a04 | 746 | |
5bcae85e | 747 | args = [a for a in sys.argv if a != '-h' and a != '--help'] |
a7813a04 XL |
748 | args, _ = parser.parse_known_args(args) |
749 | ||
750 | # Configure initial bootstrap | |
3b2f2976 | 751 | build = RustBuild() |
83c7162d | 752 | build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) |
3b2f2976 XL |
753 | build.verbose = args.verbose |
754 | build.clean = args.clean | |
a7813a04 XL |
755 | |
756 | try: | |
757 | with open(args.config or 'config.toml') as config: | |
3b2f2976 | 758 | build.config_toml = config.read() |
ea8adc8c | 759 | except (OSError, IOError): |
a7813a04 XL |
760 | pass |
761 | ||
0531ce1d XL |
762 | match = re.search(r'\nverbose = (\d+)', build.config_toml) |
763 | if match is not None: | |
764 | build.verbose = max(build.verbose, int(match.group(1))) | |
7cac9316 | 765 | |
ea8adc8c | 766 | build.use_vendored_sources = '\nvendor = true' in build.config_toml |
476ff2be | 767 | |
ea8adc8c | 768 | build.use_locked_deps = '\nlocked-deps = true' in build.config_toml |
8bb4bdeb | 769 | |
3b2f2976 | 770 | if 'SUDO_USER' in os.environ and not build.use_vendored_sources: |
32a655c1 | 771 | if os.environ.get('USER') != os.environ['SUDO_USER']: |
3b2f2976 | 772 | build.use_vendored_sources = True |
476ff2be SL |
773 | print('info: looks like you are running this command under `sudo`') |
774 | print(' and so in order to preserve your $HOME this will now') | |
775 | print(' use vendored sources by default. Note that if this') | |
776 | print(' does not work you should run a normal build first') | |
0531ce1d | 777 | print(' before running a command like `sudo ./x.py install`') |
476ff2be | 778 | |
3b2f2976 | 779 | if build.use_vendored_sources: |
476ff2be SL |
780 | if not os.path.exists('.cargo'): |
781 | os.makedirs('.cargo') | |
8faf50e0 | 782 | with output('.cargo/config') as cargo_config: |
3b2f2976 | 783 | cargo_config.write(""" |
476ff2be SL |
784 | [source.crates-io] |
785 | replace-with = 'vendored-sources' | |
786 | registry = 'https://example.com' | |
787 | ||
788 | [source.vendored-sources] | |
a1dfa0c6 | 789 | directory = '{}/vendor' |
3b2f2976 | 790 | """.format(build.rust_root)) |
476ff2be SL |
791 | else: |
792 | if os.path.exists('.cargo'): | |
793 | shutil.rmtree('.cargo') | |
794 | ||
3b2f2976 XL |
795 | data = stage0_data(build.rust_root) |
796 | build.date = data['date'] | |
797 | build.rustc_channel = data['rustc'] | |
798 | build.cargo_channel = data['cargo'] | |
799 | ||
7cac9316 | 800 | if 'dev' in data: |
3b2f2976 | 801 | build.set_dev_environment() |
7cac9316 | 802 | |
83c7162d | 803 | build.update_submodules() |
a7813a04 XL |
804 | |
805 | # Fetch/build the bootstrap | |
3b2f2976 XL |
806 | build.build = args.build or build.build_triple() |
807 | build.download_stage0() | |
a7813a04 | 808 | sys.stdout.flush() |
3b2f2976 | 809 | build.build_bootstrap() |
a7813a04 XL |
810 | sys.stdout.flush() |
811 | ||
812 | # Run the bootstrap | |
3b2f2976 | 813 | args = [build.bootstrap_binary()] |
a7813a04 XL |
814 | args.extend(sys.argv[1:]) |
815 | env = os.environ.copy() | |
3b2f2976 XL |
816 | env["BUILD"] = build.build |
817 | env["SRC"] = build.rust_root | |
a7813a04 | 818 | env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) |
041b39d2 | 819 | env["BOOTSTRAP_PYTHON"] = sys.executable |
83c7162d | 820 | env["BUILD_DIR"] = build.build_dir |
8faf50e0 XL |
821 | env["RUSTC_BOOTSTRAP"] = '1' |
822 | env["CARGO"] = build.cargo() | |
823 | env["RUSTC"] = build.rustc() | |
3b2f2976 | 824 | run(args, env=env, verbose=build.verbose) |
7cac9316 | 825 | |
a7813a04 | 826 | |
8bb4bdeb | 827 | def main(): |
3b2f2976 | 828 | """Entry point for the bootstrap process""" |
8bb4bdeb | 829 | start_time = time() |
0bf4aa26 XL |
830 | |
831 | # x.py help <cmd> ... | |
832 | if len(sys.argv) > 1 and sys.argv[1] == 'help': | |
833 | sys.argv = sys.argv[:1] + [sys.argv[2], '-h'] + sys.argv[3:] | |
834 | ||
7cac9316 XL |
835 | help_triggered = ( |
836 | '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) | |
8bb4bdeb | 837 | try: |
0531ce1d | 838 | bootstrap(help_triggered) |
cc61c64b | 839 | if not help_triggered: |
3b2f2976 XL |
840 | print("Build completed successfully in {}".format( |
841 | format_build_time(time() - start_time))) | |
842 | except (SystemExit, KeyboardInterrupt) as error: | |
843 | if hasattr(error, 'code') and isinstance(error.code, int): | |
844 | exit_code = error.code | |
8bb4bdeb XL |
845 | else: |
846 | exit_code = 1 | |
3b2f2976 | 847 | print(error) |
cc61c64b | 848 | if not help_triggered: |
3b2f2976 XL |
849 | print("Build completed unsuccessfully in {}".format( |
850 | format_build_time(time() - start_time))) | |
8bb4bdeb | 851 | sys.exit(exit_code) |
3157f602 | 852 | |
041b39d2 | 853 | |
a7813a04 XL |
854 | if __name__ == '__main__': |
855 | main() |