]>
Commit | Line | Data |
---|---|---|
abe05a73 | 1 | from __future__ import absolute_import, division, print_function |
7453a54e SL |
2 | import argparse |
3 | import contextlib | |
3157f602 | 4 | import datetime |
f9f354fc | 5 | import distutils.version |
a7813a04 | 6 | import hashlib |
7453a54e | 7 | import os |
7cac9316 | 8 | import re |
7453a54e SL |
9 | import shutil |
10 | import subprocess | |
11 | import sys | |
12 | import tarfile | |
a7813a04 XL |
13 | import tempfile |
14 | ||
3157f602 XL |
15 | from time import time |
16 | ||
1b1a35ee XL |
17 | def support_xz(): |
18 | try: | |
19 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: | |
20 | temp_path = temp_file.name | |
21 | with tarfile.open(temp_path, "w:xz"): | |
22 | pass | |
23 | return True | |
24 | except tarfile.CompressionError: | |
25 | return False | |
26 | ||
27 | def get(url, path, verbose=False, do_verify=True): | |
041b39d2 XL |
28 | suffix = '.sha256' |
29 | sha_url = url + suffix | |
a7813a04 XL |
30 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: |
31 | temp_path = temp_file.name | |
041b39d2 | 32 | with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: |
a7813a04 XL |
33 | sha_path = sha_file.name |
34 | ||
35 | try: | |
1b1a35ee XL |
36 | if do_verify: |
37 | download(sha_path, sha_url, False, verbose) | |
38 | if os.path.exists(path): | |
39 | if verify(path, sha_path, False): | |
40 | if verbose: | |
41 | print("using already-download file", path) | |
42 | return | |
43 | else: | |
44 | if verbose: | |
45 | print("ignoring already-download file", | |
46 | path, "due to failed verification") | |
47 | os.unlink(path) | |
476ff2be | 48 | download(temp_path, url, True, verbose) |
1b1a35ee | 49 | if do_verify and not verify(temp_path, sha_path, verbose): |
5bcae85e | 50 | raise RuntimeError("failed verification") |
476ff2be SL |
51 | if verbose: |
52 | print("moving {} to {}".format(temp_path, path)) | |
a7813a04 XL |
53 | shutil.move(temp_path, path) |
54 | finally: | |
476ff2be SL |
55 | delete_if_present(sha_path, verbose) |
56 | delete_if_present(temp_path, verbose) | |
a7813a04 XL |
57 | |
58 | ||
476ff2be | 59 | def delete_if_present(path, verbose): |
041b39d2 | 60 | """Remove the given file if present""" |
a7813a04 | 61 | if os.path.isfile(path): |
476ff2be | 62 | if verbose: |
3b2f2976 | 63 | print("removing", path) |
a7813a04 XL |
64 | os.unlink(path) |
65 | ||
66 | ||
476ff2be | 67 | def download(path, url, probably_big, verbose): |
3b2f2976 | 68 | for _ in range(0, 4): |
8bb4bdeb XL |
69 | try: |
70 | _download(path, url, probably_big, verbose, True) | |
71 | return | |
72 | except RuntimeError: | |
73 | print("\nspurious failure, trying again") | |
74 | _download(path, url, probably_big, verbose, False) | |
75 | ||
76 | ||
77 | def _download(path, url, probably_big, verbose, exception): | |
476ff2be SL |
78 | if probably_big or verbose: |
79 | print("downloading {}".format(url)) | |
7453a54e SL |
80 | # see http://serverfault.com/questions/301128/how-to-download |
81 | if sys.platform == 'win32': | |
82 | run(["PowerShell.exe", "/nologo", "-Command", | |
a1dfa0c6 XL |
83 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", |
84 | "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], | |
8bb4bdeb XL |
85 | verbose=verbose, |
86 | exception=exception) | |
7453a54e | 87 | else: |
476ff2be SL |
88 | if probably_big or verbose: |
89 | option = "-#" | |
90 | else: | |
91 | option = "-s" | |
f9f354fc | 92 | require(["curl", "--version"]) |
b7449926 XL |
93 | run(["curl", option, |
94 | "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds | |
74b04a01 | 95 | "--connect-timeout", "30", # timeout if cannot connect within 30 seconds |
b7449926 | 96 | "--retry", "3", "-Sf", "-o", path, url], |
8bb4bdeb XL |
97 | verbose=verbose, |
98 | exception=exception) | |
7453a54e | 99 | |
a7813a04 XL |
100 | |
101 | def verify(path, sha_path, verbose): | |
041b39d2 | 102 | """Check if the sha256 sum of the given path is valid""" |
476ff2be | 103 | if verbose: |
3b2f2976 | 104 | print("verifying", path) |
041b39d2 XL |
105 | with open(path, "rb") as source: |
106 | found = hashlib.sha256(source.read()).hexdigest() | |
107 | with open(sha_path, "r") as sha256sum: | |
108 | expected = sha256sum.readline().split()[0] | |
5bcae85e | 109 | verified = found == expected |
476ff2be | 110 | if not verified: |
5bcae85e | 111 | print("invalid checksum:\n" |
7cac9316 XL |
112 | " found: {}\n" |
113 | " expected: {}".format(found, expected)) | |
5bcae85e | 114 | return verified |
a7813a04 XL |
115 | |
116 | ||
60c5eb7d | 117 | def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): |
041b39d2 | 118 | """Unpack the given tarball file""" |
3b2f2976 | 119 | print("extracting", tarball) |
60c5eb7d | 120 | fname = os.path.basename(tarball).replace(tarball_suffix, "") |
7453a54e | 121 | with contextlib.closing(tarfile.open(tarball)) as tar: |
3b2f2976 XL |
122 | for member in tar.getnames(): |
123 | if "/" not in member: | |
7453a54e | 124 | continue |
3b2f2976 | 125 | name = member.replace(fname + "/", "", 1) |
7453a54e SL |
126 | if match is not None and not name.startswith(match): |
127 | continue | |
128 | name = name[len(match) + 1:] | |
129 | ||
3b2f2976 | 130 | dst_path = os.path.join(dst, name) |
7453a54e | 131 | if verbose: |
3b2f2976 XL |
132 | print(" extracting", member) |
133 | tar.extract(member, dst) | |
134 | src_path = os.path.join(dst, member) | |
135 | if os.path.isdir(src_path) and os.path.exists(dst_path): | |
7453a54e | 136 | continue |
3b2f2976 | 137 | shutil.move(src_path, dst_path) |
7453a54e SL |
138 | shutil.rmtree(os.path.join(dst, fname)) |
139 | ||
041b39d2 | 140 | |
7cac9316 | 141 | def run(args, verbose=False, exception=False, **kwargs): |
3b2f2976 | 142 | """Run a child program in a new process""" |
7453a54e SL |
143 | if verbose: |
144 | print("running: " + ' '.join(args)) | |
145 | sys.stdout.flush() | |
146 | # Use Popen here instead of call() as it apparently allows powershell on | |
147 | # Windows to not lock up waiting for input presumably. | |
7cac9316 | 148 | ret = subprocess.Popen(args, **kwargs) |
7453a54e SL |
149 | code = ret.wait() |
150 | if code != 0: | |
a7813a04 | 151 | err = "failed to run: " + ' '.join(args) |
8bb4bdeb | 152 | if verbose or exception: |
a7813a04 XL |
153 | raise RuntimeError(err) |
154 | sys.exit(err) | |
155 | ||
7cac9316 | 156 | |
f9f354fc XL |
157 | def require(cmd, exit=True): |
158 | '''Run a command, returning its output. | |
159 | On error, | |
160 | If `exit` is `True`, exit the process. | |
161 | Otherwise, return None.''' | |
162 | try: | |
163 | return subprocess.check_output(cmd).strip() | |
164 | except (subprocess.CalledProcessError, OSError) as exc: | |
165 | if not exit: | |
166 | return None | |
167 | print("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) | |
168 | print("Please make sure it's installed and in the path.") | |
169 | sys.exit(1) | |
170 | ||
171 | ||
a7813a04 | 172 | def stage0_data(rust_root): |
3b2f2976 | 173 | """Build a dictionary from stage0.txt""" |
a7813a04 XL |
174 | nightlies = os.path.join(rust_root, "src/stage0.txt") |
175 | with open(nightlies, 'r') as nightlies: | |
3b2f2976 XL |
176 | lines = [line.rstrip() for line in nightlies |
177 | if not line.startswith("#")] | |
178 | return dict([line.split(": ", 1) for line in lines if line]) | |
3157f602 | 179 | |
7cac9316 | 180 | |
3157f602 | 181 | def format_build_time(duration): |
3b2f2976 XL |
182 | """Return a nicer format for build time |
183 | ||
184 | >>> format_build_time('300') | |
185 | '0:05:00' | |
186 | """ | |
3157f602 | 187 | return str(datetime.timedelta(seconds=int(duration))) |
7453a54e | 188 | |
9e0c209e | 189 | |
29967ef6 | 190 | def default_build_triple(verbose): |
ea8adc8c | 191 | """Build triple as in LLVM""" |
29967ef6 XL |
192 | # If the user already has a host build triple with an existing `rustc` |
193 | # install, use their preference. This fixes most issues with Windows builds | |
194 | # being detected as GNU instead of MSVC. | |
fc512014 | 195 | default_encoding = sys.getdefaultencoding() |
29967ef6 | 196 | try: |
5869c6ff XL |
197 | version = subprocess.check_output(["rustc", "--version", "--verbose"], |
198 | stderr=subprocess.DEVNULL) | |
fc512014 | 199 | version = version.decode(default_encoding) |
29967ef6 XL |
200 | host = next(x for x in version.split('\n') if x.startswith("host: ")) |
201 | triple = host.split("host: ")[1] | |
202 | if verbose: | |
203 | print("detected default triple {}".format(triple)) | |
204 | return triple | |
205 | except Exception as e: | |
206 | if verbose: | |
207 | print("rustup not detected: {}".format(e)) | |
208 | print("falling back to auto-detect") | |
209 | ||
f9f354fc XL |
210 | required = sys.platform != 'win32' |
211 | ostype = require(["uname", "-s"], exit=required) | |
212 | cputype = require(['uname', '-m'], exit=required) | |
213 | ||
f035d41b | 214 | # If we do not have `uname`, assume Windows. |
f9f354fc XL |
215 | if ostype is None or cputype is None: |
216 | return 'x86_64-pc-windows-msvc' | |
217 | ||
218 | ostype = ostype.decode(default_encoding) | |
219 | cputype = cputype.decode(default_encoding) | |
ea8adc8c XL |
220 | |
221 | # The goal here is to come up with the same triple as LLVM would, | |
222 | # at least for the subset of platforms we're willing to target. | |
223 | ostype_mapper = { | |
ea8adc8c XL |
224 | 'Darwin': 'apple-darwin', |
225 | 'DragonFly': 'unknown-dragonfly', | |
226 | 'FreeBSD': 'unknown-freebsd', | |
227 | 'Haiku': 'unknown-haiku', | |
228 | 'NetBSD': 'unknown-netbsd', | |
229 | 'OpenBSD': 'unknown-openbsd' | |
230 | } | |
231 | ||
232 | # Consider the direct transformation first and then the special cases | |
233 | if ostype in ostype_mapper: | |
234 | ostype = ostype_mapper[ostype] | |
235 | elif ostype == 'Linux': | |
236 | os_from_sp = subprocess.check_output( | |
237 | ['uname', '-o']).strip().decode(default_encoding) | |
238 | if os_from_sp == 'Android': | |
239 | ostype = 'linux-android' | |
240 | else: | |
241 | ostype = 'unknown-linux-gnu' | |
242 | elif ostype == 'SunOS': | |
6a06907d | 243 | ostype = 'pc-solaris' |
ea8adc8c XL |
244 | # On Solaris, uname -m will return a machine classification instead |
245 | # of a cpu type, so uname -p is recommended instead. However, the | |
246 | # output from that option is too generic for our purposes (it will | |
247 | # always emit 'i386' on x86/amd64 systems). As such, isainfo -k | |
248 | # must be used instead. | |
f9f354fc | 249 | cputype = require(['isainfo', '-k']).decode(default_encoding) |
6a06907d XL |
250 | # sparc cpus have sun as a target vendor |
251 | if 'sparc' in cputype: | |
252 | ostype = 'sun-solaris' | |
ea8adc8c XL |
253 | elif ostype.startswith('MINGW'): |
254 | # msys' `uname` does not print gcc configuration, but prints msys | |
255 | # configuration. so we cannot believe `uname -m`: | |
256 | # msys1 is always i686 and msys2 is always x86_64. | |
257 | # instead, msys defines $MSYSTEM which is MINGW32 on i686 and | |
258 | # MINGW64 on x86_64. | |
259 | ostype = 'pc-windows-gnu' | |
260 | cputype = 'i686' | |
261 | if os.environ.get('MSYSTEM') == 'MINGW64': | |
262 | cputype = 'x86_64' | |
263 | elif ostype.startswith('MSYS'): | |
264 | ostype = 'pc-windows-gnu' | |
265 | elif ostype.startswith('CYGWIN_NT'): | |
266 | cputype = 'i686' | |
267 | if ostype.endswith('WOW64'): | |
268 | cputype = 'x86_64' | |
269 | ostype = 'pc-windows-gnu' | |
f035d41b XL |
270 | elif sys.platform == 'win32': |
271 | # Some Windows platforms might have a `uname` command that returns a | |
272 | # non-standard string (e.g. gnuwin32 tools returns `windows32`). In | |
273 | # these cases, fall back to using sys.platform. | |
274 | return 'x86_64-pc-windows-msvc' | |
ea8adc8c XL |
275 | else: |
276 | err = "unknown OS type: {}".format(ostype) | |
277 | sys.exit(err) | |
278 | ||
9fa01778 XL |
279 | if cputype == 'powerpc' and ostype == 'unknown-freebsd': |
280 | cputype = subprocess.check_output( | |
281 | ['uname', '-p']).strip().decode(default_encoding) | |
ea8adc8c XL |
282 | cputype_mapper = { |
283 | 'BePC': 'i686', | |
284 | 'aarch64': 'aarch64', | |
285 | 'amd64': 'x86_64', | |
286 | 'arm64': 'aarch64', | |
287 | 'i386': 'i686', | |
288 | 'i486': 'i686', | |
289 | 'i686': 'i686', | |
290 | 'i786': 'i686', | |
291 | 'powerpc': 'powerpc', | |
292 | 'powerpc64': 'powerpc64', | |
293 | 'powerpc64le': 'powerpc64le', | |
294 | 'ppc': 'powerpc', | |
295 | 'ppc64': 'powerpc64', | |
296 | 'ppc64le': 'powerpc64le', | |
297 | 's390x': 's390x', | |
298 | 'x64': 'x86_64', | |
299 | 'x86': 'i686', | |
300 | 'x86-64': 'x86_64', | |
301 | 'x86_64': 'x86_64' | |
302 | } | |
303 | ||
304 | # Consider the direct transformation first and then the special cases | |
305 | if cputype in cputype_mapper: | |
306 | cputype = cputype_mapper[cputype] | |
307 | elif cputype in {'xscale', 'arm'}: | |
308 | cputype = 'arm' | |
309 | if ostype == 'linux-android': | |
310 | ostype = 'linux-androideabi' | |
532ac7d7 XL |
311 | elif ostype == 'unknown-freebsd': |
312 | cputype = subprocess.check_output( | |
313 | ['uname', '-p']).strip().decode(default_encoding) | |
314 | ostype = 'unknown-freebsd' | |
ea8adc8c XL |
315 | elif cputype == 'armv6l': |
316 | cputype = 'arm' | |
317 | if ostype == 'linux-android': | |
318 | ostype = 'linux-androideabi' | |
319 | else: | |
320 | ostype += 'eabihf' | |
321 | elif cputype in {'armv7l', 'armv8l'}: | |
322 | cputype = 'armv7' | |
323 | if ostype == 'linux-android': | |
324 | ostype = 'linux-androideabi' | |
325 | else: | |
326 | ostype += 'eabihf' | |
327 | elif cputype == 'mips': | |
328 | if sys.byteorder == 'big': | |
329 | cputype = 'mips' | |
330 | elif sys.byteorder == 'little': | |
331 | cputype = 'mipsel' | |
332 | else: | |
333 | raise ValueError("unknown byteorder: {}".format(sys.byteorder)) | |
334 | elif cputype == 'mips64': | |
335 | if sys.byteorder == 'big': | |
336 | cputype = 'mips64' | |
337 | elif sys.byteorder == 'little': | |
338 | cputype = 'mips64el' | |
339 | else: | |
340 | raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) | |
341 | # only the n64 ABI is supported, indicate it | |
342 | ostype += 'abi64' | |
0531ce1d | 343 | elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': |
ea8adc8c XL |
344 | pass |
345 | else: | |
346 | err = "unknown cpu type: {}".format(cputype) | |
347 | sys.exit(err) | |
348 | ||
349 | return "{}-{}".format(cputype, ostype) | |
350 | ||
abe05a73 | 351 | |
8faf50e0 XL |
352 | @contextlib.contextmanager |
353 | def output(filepath): | |
354 | tmp = filepath + '.tmp' | |
355 | with open(tmp, 'w') as f: | |
356 | yield f | |
357 | try: | |
5869c6ff XL |
358 | if os.path.exists(filepath): |
359 | os.remove(filepath) # PermissionError/OSError on Win32 if in use | |
8faf50e0 XL |
360 | except OSError: |
361 | shutil.copy2(tmp, filepath) | |
362 | os.remove(tmp) | |
5869c6ff XL |
363 | return |
364 | os.rename(tmp, filepath) | |
8faf50e0 XL |
365 | |
366 | ||
9e0c209e | 367 | class RustBuild(object): |
3b2f2976 XL |
368 | """Provide all the methods required to build Rust""" |
369 | def __init__(self): | |
3b2f2976 | 370 | self.date = '' |
e1599b0c | 371 | self._download_url = '' |
3b2f2976 | 372 | self.rustc_channel = '' |
dfeec247 | 373 | self.rustfmt_channel = '' |
3b2f2976 | 374 | self.build = '' |
f9f354fc | 375 | self.build_dir = '' |
3b2f2976 | 376 | self.clean = False |
3b2f2976 | 377 | self.config_toml = '' |
83c7162d | 378 | self.rust_root = '' |
3b2f2976 XL |
379 | self.use_locked_deps = '' |
380 | self.use_vendored_sources = '' | |
381 | self.verbose = False | |
f9f354fc | 382 | self.git_version = None |
3dfed10e | 383 | self.nix_deps_dir = None |
6a06907d | 384 | self.rustc_commit = None |
7cac9316 | 385 | |
cdc7bbd5 | 386 | def download_toolchain(self, stage0=True, rustc_channel=None): |
3b2f2976 XL |
387 | """Fetch the build system for Rust, written in Rust |
388 | ||
389 | This method will build a cache directory, then it will fetch the | |
390 | tarball which has the stage0 compiler used to then bootstrap the Rust | |
391 | compiler itself. | |
cc61c64b | 392 | |
3b2f2976 XL |
393 | Each downloaded tarball is extracted, after that, the script |
394 | will move all the content to the right place. | |
395 | """ | |
cdc7bbd5 XL |
396 | if rustc_channel is None: |
397 | rustc_channel = self.rustc_channel | |
dfeec247 | 398 | rustfmt_channel = self.rustfmt_channel |
cdc7bbd5 XL |
399 | bin_root = self.bin_root(stage0) |
400 | ||
401 | key = self.date | |
402 | if not stage0: | |
403 | key += str(self.rustc_commit) | |
404 | if self.rustc(stage0).startswith(bin_root) and \ | |
405 | (not os.path.exists(self.rustc(stage0)) or | |
406 | self.program_out_of_date(self.rustc_stamp(stage0), key)): | |
407 | if os.path.exists(bin_root): | |
408 | shutil.rmtree(bin_root) | |
60c5eb7d XL |
409 | tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' |
410 | filename = "rust-std-{}-{}{}".format( | |
411 | rustc_channel, self.build, tarball_suffix) | |
3b2f2976 | 412 | pattern = "rust-std-{}".format(self.build) |
cdc7bbd5 | 413 | self._download_component_helper(filename, pattern, tarball_suffix, stage0) |
60c5eb7d XL |
414 | filename = "rustc-{}-{}{}".format(rustc_channel, self.build, |
415 | tarball_suffix) | |
cdc7bbd5 | 416 | self._download_component_helper(filename, "rustc", tarball_suffix, stage0) |
29967ef6 XL |
417 | filename = "cargo-{}-{}{}".format(rustc_channel, self.build, |
418 | tarball_suffix) | |
6a06907d | 419 | self._download_component_helper(filename, "cargo", tarball_suffix) |
cdc7bbd5 | 420 | if not stage0: |
6a06907d XL |
421 | filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix) |
422 | self._download_component_helper( | |
cdc7bbd5 | 423 | filename, "rustc-dev", tarball_suffix, stage0 |
6a06907d XL |
424 | ) |
425 | ||
cdc7bbd5 XL |
426 | self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) |
427 | self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) | |
428 | self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) | |
429 | lib_dir = "{}/lib".format(bin_root) | |
3dfed10e XL |
430 | for lib in os.listdir(lib_dir): |
431 | if lib.endswith(".so"): | |
cdc7bbd5 XL |
432 | self.fix_bin_or_dylib(os.path.join(lib_dir, lib)) |
433 | with output(self.rustc_stamp(stage0)) as rust_stamp: | |
434 | rust_stamp.write(key) | |
7453a54e | 435 | |
cdc7bbd5 | 436 | if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( |
dfeec247 | 437 | not os.path.exists(self.rustfmt()) |
74b04a01 | 438 | or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel) |
dfeec247 XL |
439 | ): |
440 | if rustfmt_channel: | |
441 | tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' | |
442 | [channel, date] = rustfmt_channel.split('-', 1) | |
443 | filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix) | |
6a06907d XL |
444 | self._download_component_helper( |
445 | filename, "rustfmt-preview", tarball_suffix, key=date | |
446 | ) | |
cdc7bbd5 XL |
447 | self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root)) |
448 | self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root)) | |
dfeec247 | 449 | with output(self.rustfmt_stamp()) as rustfmt_stamp: |
5869c6ff | 450 | rustfmt_stamp.write(self.rustfmt_channel) |
dfeec247 | 451 | |
cdc7bbd5 XL |
452 | # Avoid downloading LLVM twice (once for stage0 and once for the master rustc) |
453 | if self.downloading_llvm() and stage0: | |
1b1a35ee XL |
454 | # We want the most recent LLVM submodule update to avoid downloading |
455 | # LLVM more often than necessary. | |
456 | # | |
457 | # This git command finds that commit SHA, looking for bors-authored | |
458 | # merges that modified src/llvm-project. | |
459 | # | |
460 | # This works even in a repository that has not yet initialized | |
461 | # submodules. | |
fc512014 XL |
462 | top_level = subprocess.check_output([ |
463 | "git", "rev-parse", "--show-toplevel", | |
464 | ]).decode(sys.getdefaultencoding()).strip() | |
1b1a35ee XL |
465 | llvm_sha = subprocess.check_output([ |
466 | "git", "log", "--author=bors", "--format=%H", "-n1", | |
467 | "-m", "--first-parent", | |
29967ef6 | 468 | "--", |
fc512014 XL |
469 | "{}/src/llvm-project".format(top_level), |
470 | "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level), | |
cdc7bbd5 XL |
471 | # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` |
472 | "{}/src/version".format(top_level) | |
1b1a35ee XL |
473 | ]).decode(sys.getdefaultencoding()).strip() |
474 | llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' | |
5869c6ff XL |
475 | llvm_root = self.llvm_root() |
476 | llvm_lib = os.path.join(llvm_root, "lib") | |
1b1a35ee XL |
477 | if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): |
478 | self._download_ci_llvm(llvm_sha, llvm_assertions) | |
479 | for binary in ["llvm-config", "FileCheck"]: | |
cdc7bbd5 | 480 | self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary)) |
5869c6ff XL |
481 | for lib in os.listdir(llvm_lib): |
482 | if lib.endswith(".so"): | |
cdc7bbd5 | 483 | self.fix_bin_or_dylib(os.path.join(llvm_lib, lib)) |
1b1a35ee | 484 | with output(self.llvm_stamp()) as llvm_stamp: |
5869c6ff | 485 | llvm_stamp.write(llvm_sha + str(llvm_assertions)) |
1b1a35ee XL |
486 | |
487 | def downloading_llvm(self): | |
488 | opt = self.get_toml('download-ci-llvm', 'llvm') | |
5869c6ff XL |
489 | # This is currently all tier 1 targets (since others may not have CI |
490 | # artifacts) | |
491 | # https://doc.rust-lang.org/rustc/platform-support.html#tier-1 | |
492 | supported_platforms = [ | |
493 | "aarch64-unknown-linux-gnu", | |
494 | "i686-pc-windows-gnu", | |
495 | "i686-pc-windows-msvc", | |
496 | "i686-unknown-linux-gnu", | |
497 | "x86_64-unknown-linux-gnu", | |
498 | "x86_64-apple-darwin", | |
499 | "x86_64-pc-windows-gnu", | |
500 | "x86_64-pc-windows-msvc", | |
501 | ] | |
29967ef6 | 502 | return opt == "true" \ |
5869c6ff | 503 | or (opt == "if-available" and self.build in supported_platforms) |
1b1a35ee | 504 | |
6a06907d | 505 | def _download_component_helper( |
cdc7bbd5 | 506 | self, filename, pattern, tarball_suffix, stage0=True, key=None |
6a06907d XL |
507 | ): |
508 | if key is None: | |
cdc7bbd5 | 509 | if stage0: |
6a06907d | 510 | key = self.date |
cdc7bbd5 XL |
511 | else: |
512 | key = self.rustc_commit | |
3b2f2976 | 513 | cache_dst = os.path.join(self.build_dir, "cache") |
6a06907d | 514 | rustc_cache = os.path.join(cache_dst, key) |
3b2f2976 XL |
515 | if not os.path.exists(rustc_cache): |
516 | os.makedirs(rustc_cache) | |
517 | ||
cdc7bbd5 | 518 | if stage0: |
6a06907d | 519 | url = "{}/dist/{}".format(self._download_url, key) |
cdc7bbd5 XL |
520 | else: |
521 | url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit) | |
3b2f2976 XL |
522 | tarball = os.path.join(rustc_cache, filename) |
523 | if not os.path.exists(tarball): | |
cdc7bbd5 XL |
524 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0) |
525 | unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) | |
8bb4bdeb | 526 | |
1b1a35ee XL |
527 | def _download_ci_llvm(self, llvm_sha, llvm_assertions): |
528 | cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) | |
529 | cache_dst = os.path.join(self.build_dir, "cache") | |
530 | rustc_cache = os.path.join(cache_dst, cache_prefix) | |
531 | if not os.path.exists(rustc_cache): | |
532 | os.makedirs(rustc_cache) | |
533 | ||
534 | url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha) | |
535 | if llvm_assertions: | |
536 | url = url.replace('rustc-builds', 'rustc-builds-alt') | |
5869c6ff XL |
537 | # ci-artifacts are only stored as .xz, not .gz |
538 | if not support_xz(): | |
539 | print("error: XZ support is required to download LLVM") | |
540 | print("help: consider disabling `download-ci-llvm` or using python3") | |
541 | exit(1) | |
542 | tarball_suffix = '.tar.xz' | |
1b1a35ee XL |
543 | filename = "rust-dev-nightly-" + self.build + tarball_suffix |
544 | tarball = os.path.join(rustc_cache, filename) | |
545 | if not os.path.exists(tarball): | |
546 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False) | |
547 | unpack(tarball, tarball_suffix, self.llvm_root(), | |
548 | match="rust-dev", | |
549 | verbose=self.verbose) | |
550 | ||
cdc7bbd5 | 551 | def fix_bin_or_dylib(self, fname): |
3dfed10e XL |
552 | """Modifies the interpreter section of 'fname' to fix the dynamic linker, |
553 | or the RPATH section, to fix the dynamic library search path | |
3b2f2976 XL |
554 | |
555 | This method is only required on NixOS and uses the PatchELF utility to | |
3dfed10e | 556 | change the interpreter/RPATH of ELF executables. |
3b2f2976 XL |
557 | |
558 | Please see https://nixos.org/patchelf.html for more information | |
559 | """ | |
8bb4bdeb XL |
560 | default_encoding = sys.getdefaultencoding() |
561 | try: | |
7cac9316 XL |
562 | ostype = subprocess.check_output( |
563 | ['uname', '-s']).strip().decode(default_encoding) | |
3b2f2976 | 564 | except subprocess.CalledProcessError: |
8bb4bdeb | 565 | return |
3b2f2976 XL |
566 | except OSError as reason: |
567 | if getattr(reason, 'winerror', None) is not None: | |
568 | return | |
569 | raise reason | |
8bb4bdeb XL |
570 | |
571 | if ostype != "Linux": | |
572 | return | |
573 | ||
574 | if not os.path.exists("/etc/NIXOS"): | |
575 | return | |
576 | if os.path.exists("/lib"): | |
577 | return | |
578 | ||
579 | # At this point we're pretty sure the user is running NixOS | |
041b39d2 XL |
580 | nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" |
581 | print(nix_os_msg, fname) | |
8bb4bdeb | 582 | |
cdc7bbd5 | 583 | # Only build `.nix-deps` once. |
3dfed10e XL |
584 | nix_deps_dir = self.nix_deps_dir |
585 | if not nix_deps_dir: | |
3dfed10e XL |
586 | # Run `nix-build` to "build" each dependency (which will likely reuse |
587 | # the existing `/nix/store` copy, or at most download a pre-built copy). | |
cdc7bbd5 XL |
588 | # |
589 | # Importantly, we create a gc-root called `.nix-deps` in the `build/` | |
590 | # directory, but still reference the actual `/nix/store` path in the rpath | |
591 | # as it makes it significantly more robust against changes to the location of | |
592 | # the `.nix-deps` location. | |
593 | # | |
594 | # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). | |
595 | # zlib: Needed as a system dependency of `libLLVM-*.so`. | |
596 | # patchelf: Needed for patching ELF binaries (see doc comment above). | |
597 | nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") | |
598 | nix_expr = ''' | |
599 | with (import <nixpkgs> {}); | |
600 | symlinkJoin { | |
601 | name = "rust-stage0-dependencies"; | |
602 | paths = [ | |
603 | zlib | |
604 | patchelf | |
605 | stdenv.cc.bintools | |
606 | ]; | |
607 | } | |
608 | ''' | |
609 | try: | |
610 | subprocess.check_output([ | |
611 | "nix-build", "-E", nix_expr, "-o", nix_deps_dir, | |
612 | ]) | |
613 | except subprocess.CalledProcessError as reason: | |
614 | print("warning: failed to call nix-build:", reason) | |
615 | return | |
3dfed10e XL |
616 | self.nix_deps_dir = nix_deps_dir |
617 | ||
cdc7bbd5 XL |
618 | patchelf = "{}/bin/patchelf".format(nix_deps_dir) |
619 | rpath_entries = [ | |
620 | # Relative default, all binary and dynamic libraries we ship | |
621 | # appear to have this (even when `../lib` is redundant). | |
622 | "$ORIGIN/../lib", | |
623 | os.path.join(os.path.realpath(nix_deps_dir), "lib") | |
624 | ] | |
625 | patchelf_args = ["--set-rpath", ":".join(rpath_entries)] | |
5869c6ff XL |
626 | if not fname.endswith(".so"): |
627 | # Finally, set the corret .interp for binaries | |
cdc7bbd5 | 628 | with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: |
5869c6ff | 629 | patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] |
8bb4bdeb XL |
630 | |
631 | try: | |
3dfed10e | 632 | subprocess.check_output([patchelf] + patchelf_args + [fname]) |
3b2f2976 XL |
633 | except subprocess.CalledProcessError as reason: |
634 | print("warning: failed to call patchelf:", reason) | |
8bb4bdeb XL |
635 | return |
636 | ||
cdc7bbd5 XL |
637 | # If `download-rustc` is set, download the most recent commit with CI artifacts |
638 | def maybe_download_ci_toolchain(self): | |
6a06907d | 639 | # If `download-rustc` is not set, default to rebuilding. |
cdc7bbd5 XL |
640 | download_rustc = self.get_toml("download-rustc", section="rust") |
641 | if download_rustc is None or download_rustc == "false": | |
6a06907d | 642 | return None |
cdc7bbd5 | 643 | assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc |
6a06907d XL |
644 | |
645 | # Handle running from a directory other than the top level | |
646 | rev_parse = ["git", "rev-parse", "--show-toplevel"] | |
647 | top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip() | |
648 | compiler = "{}/compiler/".format(top_level) | |
649 | ||
650 | # Look for a version to compare to based on the current commit. | |
651 | # Only commits merged by bors will have CI artifacts. | |
652 | merge_base = ["git", "log", "--author=bors", "--pretty=%H", "-n1"] | |
653 | commit = subprocess.check_output(merge_base, universal_newlines=True).strip() | |
654 | ||
655 | # Warn if there were changes to the compiler since the ancestor commit. | |
656 | status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler]) | |
657 | if status != 0: | |
cdc7bbd5 XL |
658 | if download_rustc == "if-unchanged": |
659 | return None | |
6a06907d XL |
660 | print("warning: `download-rustc` is enabled, but there are changes to compiler/") |
661 | ||
cdc7bbd5 XL |
662 | if self.verbose: |
663 | print("using downloaded stage1 artifacts from CI (commit {})".format(commit)) | |
664 | self.rustc_commit = commit | |
665 | # FIXME: support downloading artifacts from the beta channel | |
666 | self.download_toolchain(False, "nightly") | |
6a06907d | 667 | |
cdc7bbd5 XL |
668 | def rustc_stamp(self, stage0): |
669 | """Return the path for .rustc-stamp at the given stage | |
3b2f2976 XL |
670 | |
671 | >>> rb = RustBuild() | |
672 | >>> rb.build_dir = "build" | |
cdc7bbd5 XL |
673 | >>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp") |
674 | True | |
675 | >>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp") | |
3b2f2976 XL |
676 | True |
677 | """ | |
cdc7bbd5 | 678 | return os.path.join(self.bin_root(stage0), '.rustc-stamp') |
7453a54e | 679 | |
dfeec247 XL |
680 | def rustfmt_stamp(self): |
681 | """Return the path for .rustfmt-stamp | |
682 | ||
683 | >>> rb = RustBuild() | |
684 | >>> rb.build_dir = "build" | |
685 | >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") | |
686 | True | |
687 | """ | |
cdc7bbd5 | 688 | return os.path.join(self.bin_root(True), '.rustfmt-stamp') |
dfeec247 | 689 | |
1b1a35ee XL |
690 | def llvm_stamp(self): |
691 | """Return the path for .rustfmt-stamp | |
692 | ||
693 | >>> rb = RustBuild() | |
694 | >>> rb.build_dir = "build" | |
695 | >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp") | |
696 | True | |
697 | """ | |
698 | return os.path.join(self.llvm_root(), '.llvm-stamp') | |
699 | ||
700 | ||
5869c6ff | 701 | def program_out_of_date(self, stamp_path, key): |
3b2f2976 XL |
702 | """Check if the given program stamp is out of date""" |
703 | if not os.path.exists(stamp_path) or self.clean: | |
7453a54e | 704 | return True |
3b2f2976 | 705 | with open(stamp_path, 'r') as stamp: |
5869c6ff | 706 | return key != stamp.read() |
7453a54e | 707 | |
cdc7bbd5 XL |
708 | def bin_root(self, stage0): |
709 | """Return the binary root directory for the given stage | |
3b2f2976 XL |
710 | |
711 | >>> rb = RustBuild() | |
712 | >>> rb.build_dir = "build" | |
cdc7bbd5 XL |
713 | >>> rb.bin_root(True) == os.path.join("build", "stage0") |
714 | True | |
715 | >>> rb.bin_root(False) == os.path.join("build", "ci-rustc") | |
3b2f2976 XL |
716 | True |
717 | ||
718 | When the 'build' property is given should be a nested directory: | |
719 | ||
720 | >>> rb.build = "devel" | |
cdc7bbd5 | 721 | >>> rb.bin_root(True) == os.path.join("build", "devel", "stage0") |
3b2f2976 XL |
722 | True |
723 | """ | |
cdc7bbd5 XL |
724 | if stage0: |
725 | subdir = "stage0" | |
726 | else: | |
727 | subdir = "ci-rustc" | |
728 | return os.path.join(self.build_dir, self.build, subdir) | |
7453a54e | 729 | |
1b1a35ee XL |
730 | def llvm_root(self): |
731 | """Return the CI LLVM root directory | |
732 | ||
733 | >>> rb = RustBuild() | |
734 | >>> rb.build_dir = "build" | |
735 | >>> rb.llvm_root() == os.path.join("build", "ci-llvm") | |
736 | True | |
737 | ||
738 | When the 'build' property is given should be a nested directory: | |
739 | ||
740 | >>> rb.build = "devel" | |
741 | >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm") | |
742 | True | |
743 | """ | |
744 | return os.path.join(self.build_dir, self.build, "ci-llvm") | |
745 | ||
94b46f34 | 746 | def get_toml(self, key, section=None): |
3b2f2976 XL |
747 | """Returns the value of the given key in config.toml, otherwise returns None |
748 | ||
749 | >>> rb = RustBuild() | |
750 | >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' | |
751 | >>> rb.get_toml("key2") | |
752 | 'value2' | |
753 | ||
754 | If the key does not exists, the result is None: | |
755 | ||
abe05a73 | 756 | >>> rb.get_toml("key3") is None |
3b2f2976 | 757 | True |
94b46f34 XL |
758 | |
759 | Optionally also matches the section the key appears in | |
760 | ||
761 | >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' | |
762 | >>> rb.get_toml('key', 'a') | |
763 | 'value1' | |
764 | >>> rb.get_toml('key', 'b') | |
765 | 'value2' | |
766 | >>> rb.get_toml('key', 'c') is None | |
767 | True | |
e1599b0c XL |
768 | |
769 | >>> rb.config_toml = 'key1 = true' | |
770 | >>> rb.get_toml("key1") | |
771 | 'true' | |
3b2f2976 | 772 | """ |
94b46f34 XL |
773 | |
774 | cur_section = None | |
7453a54e | 775 | for line in self.config_toml.splitlines(): |
94b46f34 XL |
776 | section_match = re.match(r'^\s*\[(.*)\]\s*$', line) |
777 | if section_match is not None: | |
778 | cur_section = section_match.group(1) | |
779 | ||
7cac9316 XL |
780 | match = re.match(r'^{}\s*=(.*)$'.format(key), line) |
781 | if match is not None: | |
782 | value = match.group(1) | |
94b46f34 XL |
783 | if section is None or section == cur_section: |
784 | return self.get_string(value) or value.strip() | |
7453a54e SL |
785 | return None |
786 | ||
7453a54e | 787 | def cargo(self): |
3b2f2976 XL |
788 | """Return config path for cargo""" |
789 | return self.program_config('cargo') | |
7453a54e | 790 | |
cdc7bbd5 | 791 | def rustc(self, stage0): |
3b2f2976 | 792 | """Return config path for rustc""" |
cdc7bbd5 | 793 | return self.program_config('rustc', stage0) |
3b2f2976 | 794 | |
dfeec247 XL |
795 | def rustfmt(self): |
796 | """Return config path for rustfmt""" | |
797 | if not self.rustfmt_channel: | |
798 | return None | |
799 | return self.program_config('rustfmt') | |
800 | ||
cdc7bbd5 XL |
801 | def program_config(self, program, stage0=True): |
802 | """Return config path for the given program at the given stage | |
3b2f2976 XL |
803 | |
804 | >>> rb = RustBuild() | |
805 | >>> rb.config_toml = 'rustc = "rustc"\\n' | |
3b2f2976 XL |
806 | >>> rb.program_config('rustc') |
807 | 'rustc' | |
3b2f2976 | 808 | >>> rb.config_toml = '' |
cdc7bbd5 XL |
809 | >>> cargo_path = rb.program_config('cargo', True) |
810 | >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True), | |
811 | ... "bin", "cargo") | |
812 | True | |
813 | >>> cargo_path = rb.program_config('cargo', False) | |
814 | >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False), | |
3b2f2976 XL |
815 | ... "bin", "cargo") |
816 | True | |
817 | """ | |
818 | config = self.get_toml(program) | |
7453a54e | 819 | if config: |
abe05a73 | 820 | return os.path.expanduser(config) |
cdc7bbd5 | 821 | return os.path.join(self.bin_root(stage0), "bin", "{}{}".format( |
3b2f2976 XL |
822 | program, self.exe_suffix())) |
823 | ||
824 | @staticmethod | |
825 | def get_string(line): | |
826 | """Return the value between double quotes | |
827 | ||
828 | >>> RustBuild.get_string(' "devel" ') | |
829 | 'devel' | |
e1599b0c XL |
830 | >>> RustBuild.get_string(" 'devel' ") |
831 | 'devel' | |
832 | >>> RustBuild.get_string('devel') is None | |
833 | True | |
834 | >>> RustBuild.get_string(' "devel ') | |
835 | '' | |
3b2f2976 | 836 | """ |
7453a54e | 837 | start = line.find('"') |
ea8adc8c XL |
838 | if start != -1: |
839 | end = start + 1 + line[start + 1:].find('"') | |
840 | return line[start + 1:end] | |
841 | start = line.find('\'') | |
842 | if start != -1: | |
843 | end = start + 1 + line[start + 1:].find('\'') | |
844 | return line[start + 1:end] | |
845 | return None | |
7453a54e | 846 | |
3b2f2976 XL |
847 | @staticmethod |
848 | def exe_suffix(): | |
849 | """Return a suffix for executables""" | |
7453a54e SL |
850 | if sys.platform == 'win32': |
851 | return '.exe' | |
041b39d2 | 852 | return '' |
7453a54e | 853 | |
476ff2be | 854 | def bootstrap_binary(self): |
0bf4aa26 | 855 | """Return the path of the bootstrap binary |
3b2f2976 XL |
856 | |
857 | >>> rb = RustBuild() | |
858 | >>> rb.build_dir = "build" | |
859 | >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", | |
860 | ... "debug", "bootstrap") | |
861 | True | |
862 | """ | |
863 | return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") | |
476ff2be | 864 | |
7453a54e | 865 | def build_bootstrap(self): |
3b2f2976 | 866 | """Build bootstrap""" |
3157f602 XL |
867 | build_dir = os.path.join(self.build_dir, "bootstrap") |
868 | if self.clean and os.path.exists(build_dir): | |
869 | shutil.rmtree(build_dir) | |
7453a54e | 870 | env = os.environ.copy() |
ba9703b0 XL |
871 | # `CARGO_BUILD_TARGET` breaks bootstrap build. |
872 | # See also: <https://github.com/rust-lang/rust/issues/70208>. | |
873 | if "CARGO_BUILD_TARGET" in env: | |
874 | del env["CARGO_BUILD_TARGET"] | |
3157f602 | 875 | env["CARGO_TARGET_DIR"] = build_dir |
cdc7bbd5 XL |
876 | env["RUSTC"] = self.rustc(True) |
877 | env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ | |
7cac9316 XL |
878 | (os.pathsep + env["LD_LIBRARY_PATH"]) \ |
879 | if "LD_LIBRARY_PATH" in env else "" | |
cdc7bbd5 | 880 | env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ |
7cac9316 XL |
881 | (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ |
882 | if "DYLD_LIBRARY_PATH" in env else "" | |
cdc7bbd5 | 883 | env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \ |
7cac9316 XL |
884 | (os.pathsep + env["LIBRARY_PATH"]) \ |
885 | if "LIBRARY_PATH" in env else "" | |
60c5eb7d XL |
886 | # preserve existing RUSTFLAGS |
887 | env.setdefault("RUSTFLAGS", "") | |
888 | env["RUSTFLAGS"] += " -Cdebuginfo=2" | |
94b46f34 | 889 | |
fc512014 | 890 | build_section = "target.{}".format(self.build) |
94b46f34 XL |
891 | target_features = [] |
892 | if self.get_toml("crt-static", build_section) == "true": | |
893 | target_features += ["+crt-static"] | |
894 | elif self.get_toml("crt-static", build_section) == "false": | |
895 | target_features += ["-crt-static"] | |
896 | if target_features: | |
60c5eb7d | 897 | env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) |
13cf67c4 XL |
898 | target_linker = self.get_toml("linker", build_section) |
899 | if target_linker is not None: | |
60c5eb7d | 900 | env["RUSTFLAGS"] += " -C linker=" + target_linker |
5869c6ff | 901 | # cfg(bootstrap): Add `-Wsemicolon_in_expressions_from_macros` after the next beta bump |
60c5eb7d | 902 | env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" |
e1599b0c | 903 | if self.get_toml("deny-warnings", "rust") != "false": |
60c5eb7d | 904 | env["RUSTFLAGS"] += " -Dwarnings" |
94b46f34 | 905 | |
cdc7bbd5 | 906 | env["PATH"] = os.path.join(self.bin_root(True), "bin") + \ |
7cac9316 | 907 | os.pathsep + env["PATH"] |
32a655c1 | 908 | if not os.path.isfile(self.cargo()): |
3b2f2976 XL |
909 | raise Exception("no cargo executable found at `{}`".format( |
910 | self.cargo())) | |
476ff2be SL |
911 | args = [self.cargo(), "build", "--manifest-path", |
912 | os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] | |
0531ce1d | 913 | for _ in range(1, self.verbose): |
7cac9316 | 914 | args.append("--verbose") |
8bb4bdeb XL |
915 | if self.use_locked_deps: |
916 | args.append("--locked") | |
476ff2be SL |
917 | if self.use_vendored_sources: |
918 | args.append("--frozen") | |
7cac9316 | 919 | run(args, env=env, verbose=self.verbose) |
7453a54e SL |
920 | |
921 | def build_triple(self): | |
fc512014 XL |
922 | """Build triple as in LLVM |
923 | ||
924 | Note that `default_build_triple` is moderately expensive, | |
925 | so use `self.build` where possible. | |
926 | """ | |
7453a54e SL |
927 | config = self.get_toml('build') |
928 | if config: | |
929 | return config | |
29967ef6 | 930 | return default_build_triple(self.verbose) |
7453a54e | 931 | |
0531ce1d XL |
932 | def check_submodule(self, module, slow_submodules): |
933 | if not slow_submodules: | |
934 | checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], | |
935 | cwd=os.path.join(self.rust_root, module), | |
936 | stdout=subprocess.PIPE) | |
937 | return checked_out | |
938 | else: | |
939 | return None | |
940 | ||
941 | def update_submodule(self, module, checked_out, recorded_submodules): | |
942 | module_path = os.path.join(self.rust_root, module) | |
943 | ||
e1599b0c | 944 | if checked_out is not None: |
0531ce1d XL |
945 | default_encoding = sys.getdefaultencoding() |
946 | checked_out = checked_out.communicate()[0].decode(default_encoding).strip() | |
947 | if recorded_submodules[module] == checked_out: | |
948 | return | |
949 | ||
950 | print("Updating submodule", module) | |
951 | ||
952 | run(["git", "submodule", "-q", "sync", module], | |
953 | cwd=self.rust_root, verbose=self.verbose) | |
f9f354fc XL |
954 | |
955 | update_args = ["git", "submodule", "update", "--init", "--recursive"] | |
956 | if self.git_version >= distutils.version.LooseVersion("2.11.0"): | |
957 | update_args.append("--progress") | |
958 | update_args.append(module) | |
959 | run(update_args, cwd=self.rust_root, verbose=self.verbose, exception=True) | |
960 | ||
0531ce1d XL |
961 | run(["git", "reset", "-q", "--hard"], |
962 | cwd=module_path, verbose=self.verbose) | |
963 | run(["git", "clean", "-qdfx"], | |
964 | cwd=module_path, verbose=self.verbose) | |
965 | ||
7cac9316 | 966 | def update_submodules(self): |
3b2f2976 | 967 | """Update submodules""" |
7cac9316 | 968 | if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ |
ea8adc8c | 969 | self.get_toml('submodules') == "false": |
7cac9316 | 970 | return |
e1599b0c | 971 | |
f9f354fc XL |
972 | default_encoding = sys.getdefaultencoding() |
973 | ||
974 | # check the existence and version of 'git' command | |
975 | git_version_str = require(['git', '--version']).split()[2].decode(default_encoding) | |
976 | self.git_version = distutils.version.LooseVersion(git_version_str) | |
e1599b0c | 977 | |
0531ce1d XL |
978 | slow_submodules = self.get_toml('fast-submodules') == "false" |
979 | start_time = time() | |
980 | if slow_submodules: | |
981 | print('Unconditionally updating all submodules') | |
982 | else: | |
983 | print('Updating only changed submodules') | |
7cac9316 | 984 | default_encoding = sys.getdefaultencoding() |
7cac9316 | 985 | submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( |
041b39d2 XL |
986 | ["git", "config", "--file", |
987 | os.path.join(self.rust_root, ".gitmodules"), | |
7cac9316 XL |
988 | "--get-regexp", "path"] |
989 | ).decode(default_encoding).splitlines()] | |
2c00a5a8 | 990 | filtered_submodules = [] |
0531ce1d | 991 | submodules_names = [] |
29967ef6 | 992 | llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git")) |
fc512014 XL |
993 | external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm() |
994 | llvm_needed = not self.get_toml('codegen-backends', 'rust') \ | |
995 | or "llvm" in self.get_toml('codegen-backends', 'rust') | |
2c00a5a8 | 996 | for module in submodules: |
9fa01778 | 997 | if module.endswith("llvm-project"): |
fc512014 XL |
998 | # Don't sync the llvm-project submodule if an external LLVM was |
999 | # provided, if we are downloading LLVM or if the LLVM backend is | |
1000 | # not being built. Also, if the submodule has been initialized | |
1001 | # already, sync it anyways so that it doesn't mess up contributor | |
1002 | # pull requests. | |
1003 | if external_llvm_provided or not llvm_needed: | |
29967ef6 | 1004 | if self.get_toml('lld') != 'true' and not llvm_checked_out: |
1b1a35ee | 1005 | continue |
0531ce1d XL |
1006 | check = self.check_submodule(module, slow_submodules) |
1007 | filtered_submodules.append((module, check)) | |
1008 | submodules_names.append(module) | |
1009 | recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, | |
1010 | cwd=self.rust_root, stdout=subprocess.PIPE) | |
1011 | recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() | |
1012 | recorded_submodules = {} | |
1013 | for data in recorded: | |
1014 | data = data.split() | |
1015 | recorded_submodules[data[3]] = data[2] | |
1016 | for module in filtered_submodules: | |
1017 | self.update_submodule(module[0], module[1], recorded_submodules) | |
1018 | print("Submodules updated in %.2f seconds" % (time() - start_time)) | |
7cac9316 | 1019 | |
e1599b0c XL |
1020 | def set_normal_environment(self): |
1021 | """Set download URL for normal environment""" | |
1022 | if 'RUSTUP_DIST_SERVER' in os.environ: | |
1023 | self._download_url = os.environ['RUSTUP_DIST_SERVER'] | |
1024 | else: | |
1025 | self._download_url = 'https://static.rust-lang.org' | |
1026 | ||
3b2f2976 XL |
1027 | def set_dev_environment(self): |
1028 | """Set download URL for development environment""" | |
e1599b0c XL |
1029 | if 'RUSTUP_DEV_DIST_SERVER' in os.environ: |
1030 | self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] | |
1031 | else: | |
1032 | self._download_url = 'https://dev-static.rust-lang.org' | |
3b2f2976 | 1033 | |
416331ca XL |
1034 | def check_vendored_status(self): |
1035 | """Check that vendoring is configured properly""" | |
1036 | vendor_dir = os.path.join(self.rust_root, 'vendor') | |
1037 | if 'SUDO_USER' in os.environ and not self.use_vendored_sources: | |
1038 | if os.environ.get('USER') != os.environ['SUDO_USER']: | |
1039 | self.use_vendored_sources = True | |
1040 | print('info: looks like you are running this command under `sudo`') | |
1041 | print(' and so in order to preserve your $HOME this will now') | |
1042 | print(' use vendored sources by default.') | |
1043 | if not os.path.exists(vendor_dir): | |
1044 | print('error: vendoring required, but vendor directory does not exist.') | |
1045 | print(' Run `cargo vendor` without sudo to initialize the ' | |
74b04a01 | 1046 | 'vendor directory.') |
416331ca XL |
1047 | raise Exception("{} not found".format(vendor_dir)) |
1048 | ||
1049 | if self.use_vendored_sources: | |
1050 | if not os.path.exists('.cargo'): | |
1051 | os.makedirs('.cargo') | |
1052 | with output('.cargo/config') as cargo_config: | |
1053 | cargo_config.write( | |
1054 | "[source.crates-io]\n" | |
1055 | "replace-with = 'vendored-sources'\n" | |
1056 | "registry = 'https://example.com'\n" | |
1057 | "\n" | |
1058 | "[source.vendored-sources]\n" | |
1059 | "directory = '{}/vendor'\n" | |
74b04a01 | 1060 | .format(self.rust_root)) |
416331ca XL |
1061 | else: |
1062 | if os.path.exists('.cargo'): | |
1063 | shutil.rmtree('.cargo') | |
1064 | ||
1065 | def ensure_vendored(self): | |
1066 | """Ensure that the vendored sources are available if needed""" | |
1067 | vendor_dir = os.path.join(self.rust_root, 'vendor') | |
1068 | # Note that this does not handle updating the vendored dependencies if | |
1069 | # the rust git repository is updated. Normal development usually does | |
1070 | # not use vendoring, so hopefully this isn't too much of a problem. | |
1071 | if self.use_vendored_sources and not os.path.exists(vendor_dir): | |
29967ef6 XL |
1072 | run([ |
1073 | self.cargo(), | |
1074 | "vendor", | |
1075 | "--sync=./src/tools/rust-analyzer/Cargo.toml", | |
1076 | "--sync=./compiler/rustc_codegen_cranelift/Cargo.toml", | |
1077 | ], verbose=self.verbose, cwd=self.rust_root) | |
416331ca | 1078 | |
7cac9316 | 1079 | |
0531ce1d | 1080 | def bootstrap(help_triggered): |
3b2f2976 | 1081 | """Configure, fetch, build and run the initial bootstrap""" |
0531ce1d XL |
1082 | |
1083 | # If the user is asking for help, let them know that the whole download-and-build | |
1084 | # process has to happen before anything is printed out. | |
1085 | if help_triggered: | |
1086 | print("info: Downloading and building bootstrap before processing --help") | |
1087 | print(" command. See src/bootstrap/README.md for help with common") | |
1088 | print(" commands.") | |
1089 | ||
a7813a04 XL |
1090 | parser = argparse.ArgumentParser(description='Build rust') |
1091 | parser.add_argument('--config') | |
3b2f2976 | 1092 | parser.add_argument('--build') |
3157f602 | 1093 | parser.add_argument('--clean', action='store_true') |
0531ce1d | 1094 | parser.add_argument('-v', '--verbose', action='count', default=0) |
a7813a04 | 1095 | |
5bcae85e | 1096 | args = [a for a in sys.argv if a != '-h' and a != '--help'] |
a7813a04 XL |
1097 | args, _ = parser.parse_known_args(args) |
1098 | ||
1099 | # Configure initial bootstrap | |
3b2f2976 | 1100 | build = RustBuild() |
1b1a35ee | 1101 | build.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) |
3b2f2976 XL |
1102 | build.verbose = args.verbose |
1103 | build.clean = args.clean | |
a7813a04 | 1104 | |
f035d41b XL |
1105 | # Read from `RUST_BOOTSTRAP_CONFIG`, then `--config`, then fallback to `config.toml` (if it |
1106 | # exists). | |
1107 | toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config | |
1108 | if not toml_path and os.path.exists('config.toml'): | |
1109 | toml_path = 'config.toml' | |
1110 | ||
1111 | if toml_path: | |
f9f354fc XL |
1112 | if not os.path.exists(toml_path): |
1113 | toml_path = os.path.join(build.rust_root, toml_path) | |
1114 | ||
1115 | with open(toml_path) as config: | |
3b2f2976 | 1116 | build.config_toml = config.read() |
a7813a04 | 1117 | |
29967ef6 XL |
1118 | profile = build.get_toml('profile') |
1119 | if profile is not None: | |
1120 | include_file = 'config.{}.toml'.format(profile) | |
1121 | include_dir = os.path.join(build.rust_root, 'src', 'bootstrap', 'defaults') | |
1122 | include_path = os.path.join(include_dir, include_file) | |
1123 | # HACK: This works because `build.get_toml()` returns the first match it finds for a | |
1124 | # specific key, so appending our defaults at the end allows the user to override them | |
1125 | with open(include_path) as included_toml: | |
1126 | build.config_toml += os.linesep + included_toml.read() | |
1127 | ||
e1599b0c XL |
1128 | config_verbose = build.get_toml('verbose', 'build') |
1129 | if config_verbose is not None: | |
1130 | build.verbose = max(build.verbose, int(config_verbose)) | |
7cac9316 | 1131 | |
e1599b0c | 1132 | build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' |
476ff2be | 1133 | |
e1599b0c | 1134 | build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' |
8bb4bdeb | 1135 | |
416331ca | 1136 | build.check_vendored_status() |
476ff2be | 1137 | |
f9f354fc XL |
1138 | build_dir = build.get_toml('build-dir', 'build') or 'build' |
1139 | build.build_dir = os.path.abspath(build_dir.replace("$ROOT", build.rust_root)) | |
1140 | ||
3b2f2976 XL |
1141 | data = stage0_data(build.rust_root) |
1142 | build.date = data['date'] | |
1143 | build.rustc_channel = data['rustc'] | |
3b2f2976 | 1144 | |
dfeec247 XL |
1145 | if "rustfmt" in data: |
1146 | build.rustfmt_channel = data['rustfmt'] | |
1147 | ||
7cac9316 | 1148 | if 'dev' in data: |
3b2f2976 | 1149 | build.set_dev_environment() |
e1599b0c XL |
1150 | else: |
1151 | build.set_normal_environment() | |
7cac9316 | 1152 | |
5869c6ff | 1153 | build.build = args.build or build.build_triple() |
83c7162d | 1154 | build.update_submodules() |
a7813a04 XL |
1155 | |
1156 | # Fetch/build the bootstrap | |
cdc7bbd5 XL |
1157 | build.download_toolchain() |
1158 | # Download the master compiler if `download-rustc` is set | |
1159 | build.maybe_download_ci_toolchain() | |
a7813a04 | 1160 | sys.stdout.flush() |
416331ca | 1161 | build.ensure_vendored() |
3b2f2976 | 1162 | build.build_bootstrap() |
a7813a04 XL |
1163 | sys.stdout.flush() |
1164 | ||
1165 | # Run the bootstrap | |
3b2f2976 | 1166 | args = [build.bootstrap_binary()] |
a7813a04 XL |
1167 | args.extend(sys.argv[1:]) |
1168 | env = os.environ.copy() | |
1169 | env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) | |
041b39d2 | 1170 | env["BOOTSTRAP_PYTHON"] = sys.executable |
83c7162d | 1171 | env["BUILD_DIR"] = build.build_dir |
8faf50e0 | 1172 | env["RUSTC_BOOTSTRAP"] = '1' |
f035d41b XL |
1173 | if toml_path: |
1174 | env["BOOTSTRAP_CONFIG"] = toml_path | |
cdc7bbd5 XL |
1175 | if build.rustc_commit is not None: |
1176 | env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1' | |
3b2f2976 | 1177 | run(args, env=env, verbose=build.verbose) |
7cac9316 | 1178 | |
a7813a04 | 1179 | |
8bb4bdeb | 1180 | def main(): |
3b2f2976 | 1181 | """Entry point for the bootstrap process""" |
8bb4bdeb | 1182 | start_time = time() |
0bf4aa26 XL |
1183 | |
1184 | # x.py help <cmd> ... | |
1185 | if len(sys.argv) > 1 and sys.argv[1] == 'help': | |
532ac7d7 | 1186 | sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] |
0bf4aa26 | 1187 | |
7cac9316 XL |
1188 | help_triggered = ( |
1189 | '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) | |
8bb4bdeb | 1190 | try: |
0531ce1d | 1191 | bootstrap(help_triggered) |
cc61c64b | 1192 | if not help_triggered: |
3b2f2976 XL |
1193 | print("Build completed successfully in {}".format( |
1194 | format_build_time(time() - start_time))) | |
1195 | except (SystemExit, KeyboardInterrupt) as error: | |
1196 | if hasattr(error, 'code') and isinstance(error.code, int): | |
1197 | exit_code = error.code | |
8bb4bdeb XL |
1198 | else: |
1199 | exit_code = 1 | |
3b2f2976 | 1200 | print(error) |
cc61c64b | 1201 | if not help_triggered: |
3b2f2976 XL |
1202 | print("Build completed unsuccessfully in {}".format( |
1203 | format_build_time(time() - start_time))) | |
8bb4bdeb | 1204 | sys.exit(exit_code) |
3157f602 | 1205 | |
041b39d2 | 1206 | |
a7813a04 XL |
1207 | if __name__ == '__main__': |
1208 | main() |