]>
git.proxmox.com Git - rustc.git/blob - src/bootstrap/bootstrap.py
1 # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
11 from __future__
import print_function
26 def get(url
, path
, verbose
=False):
27 sha_url
= url
+ ".sha256"
28 with tempfile
.NamedTemporaryFile(delete
=False) as temp_file
:
29 temp_path
= temp_file
.name
30 with tempfile
.NamedTemporaryFile(suffix
=".sha256", delete
=False) as sha_file
:
31 sha_path
= sha_file
.name
34 download(sha_path
, sha_url
, False, verbose
)
35 if os
.path
.exists(path
):
36 if verify(path
, sha_path
, False):
38 print("using already-download file " + path
)
42 print("ignoring already-download file " + path
+ " due to failed verification")
44 download(temp_path
, url
, True, verbose
)
45 if not verify(temp_path
, sha_path
, verbose
):
46 raise RuntimeError("failed verification")
48 print("moving {} to {}".format(temp_path
, path
))
49 shutil
.move(temp_path
, path
)
51 delete_if_present(sha_path
, verbose
)
52 delete_if_present(temp_path
, verbose
)
55 def delete_if_present(path
, verbose
):
56 if os
.path
.isfile(path
):
58 print("removing " + path
)
62 def download(path
, url
, probably_big
, verbose
):
65 _download(path
, url
, probably_big
, verbose
, True)
68 print("\nspurious failure, trying again")
69 _download(path
, url
, probably_big
, verbose
, False)
72 def _download(path
, url
, probably_big
, verbose
, exception
):
73 if probably_big
or verbose
:
74 print("downloading {}".format(url
))
75 # see http://serverfault.com/questions/301128/how-to-download
76 if sys
.platform
== 'win32':
77 run(["PowerShell.exe", "/nologo", "-Command",
78 "(New-Object System.Net.WebClient)"
79 ".DownloadFile('{}', '{}')".format(url
, path
)],
83 if probably_big
or verbose
:
87 run(["curl", option
, "--retry", "3", "-Sf", "-o", path
, url
],
92 def verify(path
, sha_path
, verbose
):
94 print("verifying " + path
)
95 with
open(path
, "rb") as f
:
96 found
= hashlib
.sha256(f
.read()).hexdigest()
97 with
open(sha_path
, "r") as f
:
98 expected
= f
.readline().split()[0]
99 verified
= found
== expected
101 print("invalid checksum:\n"
103 " expected: {}".format(found
, expected
))
107 def unpack(tarball
, dst
, verbose
=False, match
=None):
108 print("extracting " + tarball
)
109 fname
= os
.path
.basename(tarball
).replace(".tar.gz", "")
110 with contextlib
.closing(tarfile
.open(tarball
)) as tar
:
111 for p
in tar
.getnames():
114 name
= p
.replace(fname
+ "/", "", 1)
115 if match
is not None and not name
.startswith(match
):
117 name
= name
[len(match
) + 1:]
119 fp
= os
.path
.join(dst
, name
)
121 print(" extracting " + p
)
123 tp
= os
.path
.join(dst
, p
)
124 if os
.path
.isdir(tp
) and os
.path
.exists(fp
):
127 shutil
.rmtree(os
.path
.join(dst
, fname
))
129 def run(args
, verbose
=False, exception
=False):
131 print("running: " + ' '.join(args
))
133 # Use Popen here instead of call() as it apparently allows powershell on
134 # Windows to not lock up waiting for input presumably.
135 ret
= subprocess
.Popen(args
)
138 err
= "failed to run: " + ' '.join(args
)
139 if verbose
or exception
:
140 raise RuntimeError(err
)
143 def stage0_data(rust_root
):
144 nightlies
= os
.path
.join(rust_root
, "src/stage0.txt")
146 with
open(nightlies
, 'r') as nightlies
:
147 for line
in nightlies
:
148 line
= line
.rstrip() # Strip newline character, '\n'
149 if line
.startswith("#") or line
== '':
151 a
, b
= line
.split(": ", 1)
155 def format_build_time(duration
):
156 return str(datetime
.timedelta(seconds
=int(duration
)))
159 class RustBuild(object):
160 def download_stage0(self
):
161 cache_dst
= os
.path
.join(self
.build_dir
, "cache")
162 rustc_cache
= os
.path
.join(cache_dst
, self
.stage0_rustc_date())
163 cargo_cache
= os
.path
.join(cache_dst
, self
.stage0_cargo_rev())
164 if not os
.path
.exists(rustc_cache
):
165 os
.makedirs(rustc_cache
)
166 if not os
.path
.exists(cargo_cache
):
167 os
.makedirs(cargo_cache
)
169 if self
.rustc().startswith(self
.bin_root()) and \
170 (not os
.path
.exists(self
.rustc()) or self
.rustc_out_of_date()):
171 self
.print_what_it_means_to_bootstrap()
172 if os
.path
.exists(self
.bin_root()):
173 shutil
.rmtree(self
.bin_root())
174 channel
= self
.stage0_rustc_channel()
175 filename
= "rust-std-{}-{}.tar.gz".format(channel
, self
.build
)
176 url
= "https://static.rust-lang.org/dist/" + self
.stage0_rustc_date()
177 tarball
= os
.path
.join(rustc_cache
, filename
)
178 if not os
.path
.exists(tarball
):
179 get("{}/{}".format(url
, filename
), tarball
, verbose
=self
.verbose
)
180 unpack(tarball
, self
.bin_root(),
181 match
="rust-std-" + self
.build
,
182 verbose
=self
.verbose
)
184 filename
= "rustc-{}-{}.tar.gz".format(channel
, self
.build
)
185 url
= "https://static.rust-lang.org/dist/" + self
.stage0_rustc_date()
186 tarball
= os
.path
.join(rustc_cache
, filename
)
187 if not os
.path
.exists(tarball
):
188 get("{}/{}".format(url
, filename
), tarball
, verbose
=self
.verbose
)
189 unpack(tarball
, self
.bin_root(), match
="rustc", verbose
=self
.verbose
)
190 self
.fix_executable(self
.bin_root() + "/bin/rustc")
191 self
.fix_executable(self
.bin_root() + "/bin/rustdoc")
192 with
open(self
.rustc_stamp(), 'w') as f
:
193 f
.write(self
.stage0_rustc_date())
195 if self
.cargo().startswith(self
.bin_root()) and \
196 (not os
.path
.exists(self
.cargo()) or self
.cargo_out_of_date()):
197 self
.print_what_it_means_to_bootstrap()
198 filename
= "cargo-nightly-{}.tar.gz".format(self
.build
)
199 url
= "https://s3.amazonaws.com/rust-lang-ci/cargo-builds/" + self
.stage0_cargo_rev()
200 tarball
= os
.path
.join(cargo_cache
, filename
)
201 if not os
.path
.exists(tarball
):
202 get("{}/{}".format(url
, filename
), tarball
, verbose
=self
.verbose
)
203 unpack(tarball
, self
.bin_root(), match
="cargo", verbose
=self
.verbose
)
204 self
.fix_executable(self
.bin_root() + "/bin/cargo")
205 with
open(self
.cargo_stamp(), 'w') as f
:
206 f
.write(self
.stage0_cargo_rev())
208 def fix_executable(self
, fname
):
209 # If we're on NixOS we need to change the path to the dynamic loader
211 default_encoding
= sys
.getdefaultencoding()
213 ostype
= subprocess
.check_output(['uname', '-s']).strip().decode(default_encoding
)
214 except (subprocess
.CalledProcessError
, WindowsError):
217 if ostype
!= "Linux":
220 if not os
.path
.exists("/etc/NIXOS"):
222 if os
.path
.exists("/lib"):
225 # At this point we're pretty sure the user is running NixOS
226 print("info: you seem to be running NixOS. Attempting to patch " + fname
)
229 interpreter
= subprocess
.check_output(["patchelf", "--print-interpreter", fname
])
230 interpreter
= interpreter
.strip().decode(default_encoding
)
231 except subprocess
.CalledProcessError
as e
:
232 print("warning: failed to call patchelf: %s" % e
)
235 loader
= interpreter
.split("/")[-1]
238 ldd_output
= subprocess
.check_output(['ldd', '/run/current-system/sw/bin/sh'])
239 ldd_output
= ldd_output
.strip().decode(default_encoding
)
240 except subprocess
.CalledProcessError
as e
:
241 print("warning: unable to call ldd: %s" % e
)
244 for line
in ldd_output
.splitlines():
245 libname
= line
.split()[0]
246 if libname
.endswith(loader
):
247 loader_path
= libname
[:len(libname
) - len(loader
)]
250 print("warning: unable to find the path to the dynamic linker")
253 correct_interpreter
= loader_path
+ loader
256 subprocess
.check_output(["patchelf", "--set-interpreter", correct_interpreter
, fname
])
257 except subprocess
.CalledProcessError
as e
:
258 print("warning: failed to call patchelf: %s" % e
)
261 def stage0_cargo_rev(self
):
262 return self
._cargo
_rev
264 def stage0_rustc_date(self
):
265 return self
._rustc
_date
267 def stage0_rustc_channel(self
):
268 return self
._rustc
_channel
270 def rustc_stamp(self
):
271 return os
.path
.join(self
.bin_root(), '.rustc-stamp')
273 def cargo_stamp(self
):
274 return os
.path
.join(self
.bin_root(), '.cargo-stamp')
276 def rustc_out_of_date(self
):
277 if not os
.path
.exists(self
.rustc_stamp()) or self
.clean
:
279 with
open(self
.rustc_stamp(), 'r') as f
:
280 return self
.stage0_rustc_date() != f
.read()
282 def cargo_out_of_date(self
):
283 if not os
.path
.exists(self
.cargo_stamp()) or self
.clean
:
285 with
open(self
.cargo_stamp(), 'r') as f
:
286 return self
.stage0_cargo_rev() != f
.read()
289 return os
.path
.join(self
.build_dir
, self
.build
, "stage0")
291 def get_toml(self
, key
):
292 for line
in self
.config_toml
.splitlines():
293 if line
.startswith(key
+ ' ='):
294 return self
.get_string(line
)
297 def get_mk(self
, key
):
298 for line
in iter(self
.config_mk
.splitlines()):
299 if line
.startswith(key
+ ' '):
300 var
= line
[line
.find(':=') + 2:].strip()
306 config
= self
.get_toml('cargo')
309 config
= self
.get_mk('CFG_LOCAL_RUST_ROOT')
311 return config
+ '/bin/cargo' + self
.exe_suffix()
312 return os
.path
.join(self
.bin_root(), "bin/cargo" + self
.exe_suffix())
315 config
= self
.get_toml('rustc')
318 config
= self
.get_mk('CFG_LOCAL_RUST_ROOT')
320 return config
+ '/bin/rustc' + self
.exe_suffix()
321 return os
.path
.join(self
.bin_root(), "bin/rustc" + self
.exe_suffix())
323 def get_string(self
, line
):
324 start
= line
.find('"')
325 end
= start
+ 1 + line
[start
+ 1:].find('"')
326 return line
[start
+ 1:end
]
328 def exe_suffix(self
):
329 if sys
.platform
== 'win32':
334 def print_what_it_means_to_bootstrap(self
):
335 if hasattr(self
, 'printed'):
338 if os
.path
.exists(self
.bootstrap_binary()):
340 if not '--help' in sys
.argv
or len(sys
.argv
) == 1:
343 print('info: the build system for Rust is written in Rust, so this')
344 print(' script is now going to download a stage0 rust compiler')
345 print(' and then compile the build system itself')
347 print('info: in the meantime you can read more about rustbuild at')
348 print(' src/bootstrap/README.md before the download finishes')
350 def bootstrap_binary(self
):
351 return os
.path
.join(self
.build_dir
, "bootstrap/debug/bootstrap")
353 def build_bootstrap(self
):
354 self
.print_what_it_means_to_bootstrap()
355 build_dir
= os
.path
.join(self
.build_dir
, "bootstrap")
356 if self
.clean
and os
.path
.exists(build_dir
):
357 shutil
.rmtree(build_dir
)
358 env
= os
.environ
.copy()
359 env
["CARGO_TARGET_DIR"] = build_dir
360 env
["RUSTC"] = self
.rustc()
361 env
["LD_LIBRARY_PATH"] = os
.path
.join(self
.bin_root(), "lib") + \
362 (os
.pathsep
+ env
["LD_LIBRARY_PATH"]) \
363 if "LD_LIBRARY_PATH" in env
else ""
364 env
["DYLD_LIBRARY_PATH"] = os
.path
.join(self
.bin_root(), "lib") + \
365 (os
.pathsep
+ env
["DYLD_LIBRARY_PATH"]) \
366 if "DYLD_LIBRARY_PATH" in env
else ""
367 env
["PATH"] = os
.path
.join(self
.bin_root(), "bin") + \
368 os
.pathsep
+ env
["PATH"]
369 if not os
.path
.isfile(self
.cargo()):
370 raise Exception("no cargo executable found at `%s`" % self
.cargo())
371 args
= [self
.cargo(), "build", "--manifest-path",
372 os
.path
.join(self
.rust_root
, "src/bootstrap/Cargo.toml")]
373 if self
.use_locked_deps
:
374 args
.append("--locked")
375 if self
.use_vendored_sources
:
376 args
.append("--frozen")
379 def run(self
, args
, env
):
380 proc
= subprocess
.Popen(args
, env
=env
)
385 def build_triple(self
):
386 default_encoding
= sys
.getdefaultencoding()
387 config
= self
.get_toml('build')
390 config
= self
.get_mk('CFG_BUILD')
394 ostype
= subprocess
.check_output(['uname', '-s']).strip().decode(default_encoding
)
395 cputype
= subprocess
.check_output(['uname', '-m']).strip().decode(default_encoding
)
396 except (subprocess
.CalledProcessError
, OSError):
397 if sys
.platform
== 'win32':
398 return 'x86_64-pc-windows-msvc'
399 err
= "uname not found"
404 # Darwin's `uname -s` lies and always returns i386. We have to use
406 if ostype
== 'Darwin' and cputype
== 'i686':
407 args
= ['sysctl', 'hw.optional.x86_64']
408 sysctl
= subprocess
.check_output(args
).decode(default_encoding
)
412 # The goal here is to come up with the same triple as LLVM would,
413 # at least for the subset of platforms we're willing to target.
414 if ostype
== 'Linux':
415 ostype
= 'unknown-linux-gnu'
416 elif ostype
== 'FreeBSD':
417 ostype
= 'unknown-freebsd'
418 elif ostype
== 'DragonFly':
419 ostype
= 'unknown-dragonfly'
420 elif ostype
== 'Bitrig':
421 ostype
= 'unknown-bitrig'
422 elif ostype
== 'OpenBSD':
423 ostype
= 'unknown-openbsd'
424 elif ostype
== 'NetBSD':
425 ostype
= 'unknown-netbsd'
426 elif ostype
== 'SunOS':
427 ostype
= 'sun-solaris'
428 # On Solaris, uname -m will return a machine classification instead
429 # of a cpu type, so uname -p is recommended instead. However, the
430 # output from that option is too generic for our purposes (it will
431 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
432 # must be used instead.
434 cputype
= subprocess
.check_output(['isainfo',
435 '-k']).strip().decode(default_encoding
)
436 except (subprocess
.CalledProcessError
, OSError):
437 err
= "isainfo not found"
441 elif ostype
== 'Darwin':
442 ostype
= 'apple-darwin'
443 elif ostype
== 'Haiku':
444 ostype
= 'unknown-haiku'
445 elif ostype
.startswith('MINGW'):
446 # msys' `uname` does not print gcc configuration, but prints msys
447 # configuration. so we cannot believe `uname -m`:
448 # msys1 is always i686 and msys2 is always x86_64.
449 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
451 ostype
= 'pc-windows-gnu'
453 if os
.environ
.get('MSYSTEM') == 'MINGW64':
455 elif ostype
.startswith('MSYS'):
456 ostype
= 'pc-windows-gnu'
457 elif ostype
.startswith('CYGWIN_NT'):
459 if ostype
.endswith('WOW64'):
461 ostype
= 'pc-windows-gnu'
463 err
= "unknown OS type: " + ostype
465 raise ValueError(err
)
468 if cputype
in {'i386', 'i486', 'i686', 'i786', 'x86'}:
470 elif cputype
in {'xscale', 'arm'}:
472 elif cputype
in {'armv6l', 'armv7l', 'armv8l'}:
475 elif cputype
== 'armv7l':
478 elif cputype
== 'aarch64':
480 elif cputype
== 'arm64':
482 elif cputype
== 'mips':
483 if sys
.byteorder
== 'big':
485 elif sys
.byteorder
== 'little':
488 raise ValueError('unknown byteorder: ' + sys
.byteorder
)
489 elif cputype
== 'mips64':
490 if sys
.byteorder
== 'big':
492 elif sys
.byteorder
== 'little':
495 raise ValueError('unknown byteorder: ' + sys
.byteorder
)
496 # only the n64 ABI is supported, indicate it
498 elif cputype
in {'powerpc', 'ppc'}:
500 elif cputype
in {'powerpc64', 'ppc64'}:
501 cputype
= 'powerpc64'
502 elif cputype
in {'powerpc64le', 'ppc64le'}:
503 cputype
= 'powerpc64le'
504 elif cputype
== 'sparcv9':
506 elif cputype
in {'amd64', 'x86_64', 'x86-64', 'x64'}:
508 elif cputype
== 's390x':
510 elif cputype
== 'BePC':
513 err
= "unknown cpu type: " + cputype
515 raise ValueError(err
)
518 return "{}-{}".format(cputype
, ostype
)
521 parser
= argparse
.ArgumentParser(description
='Build rust')
522 parser
.add_argument('--config')
523 parser
.add_argument('--clean', action
='store_true')
524 parser
.add_argument('-v', '--verbose', action
='store_true')
526 args
= [a
for a
in sys
.argv
if a
!= '-h' and a
!= '--help']
527 args
, _
= parser
.parse_known_args(args
)
529 # Configure initial bootstrap
533 rb
.rust_root
= os
.path
.abspath(os
.path
.join(__file__
, '../../..'))
534 rb
.build_dir
= os
.path
.join(os
.getcwd(), "build")
535 rb
.verbose
= args
.verbose
536 rb
.clean
= args
.clean
539 with
open(args
.config
or 'config.toml') as config
:
540 rb
.config_toml
= config
.read()
544 rb
.config_mk
= open('config.mk').read()
548 rb
.use_vendored_sources
= '\nvendor = true' in rb
.config_toml
or \
549 'CFG_ENABLE_VENDOR' in rb
.config_mk
551 rb
.use_locked_deps
= '\nlocked-deps = true' in rb
.config_toml
or \
552 'CFG_ENABLE_LOCKED_DEPS' in rb
.config_mk
554 if 'SUDO_USER' in os
.environ
and not rb
.use_vendored_sources
:
555 if os
.environ
.get('USER') != os
.environ
['SUDO_USER']:
556 rb
.use_vendored_sources
= True
557 print('info: looks like you are running this command under `sudo`')
558 print(' and so in order to preserve your $HOME this will now')
559 print(' use vendored sources by default. Note that if this')
560 print(' does not work you should run a normal build first')
561 print(' before running a command like `sudo make install`')
563 if rb
.use_vendored_sources
:
564 if not os
.path
.exists('.cargo'):
565 os
.makedirs('.cargo')
566 with
open('.cargo/config','w') as f
:
569 replace-with = 'vendored-sources'
570 registry = 'https://example.com'
572 [source.vendored-sources]
573 directory = '{}/src/vendor'
574 """.format(rb
.rust_root
))
576 if os
.path
.exists('.cargo'):
577 shutil
.rmtree('.cargo')
579 data
= stage0_data(rb
.rust_root
)
580 rb
._rustc
_channel
, rb
._rustc
_date
= data
['rustc'].split('-', 1)
581 rb
._cargo
_rev
= data
['cargo']
583 # Fetch/build the bootstrap
584 rb
.build
= rb
.build_triple()
591 args
= [rb
.bootstrap_binary()]
592 args
.extend(sys
.argv
[1:])
593 env
= os
.environ
.copy()
594 env
["BUILD"] = rb
.build
595 env
["SRC"] = rb
.rust_root
596 env
["BOOTSTRAP_PARENT_ID"] = str(os
.getpid())
603 print("Build completed successfully in %s" % format_build_time(time() - start_time
))
604 except (SystemExit, KeyboardInterrupt) as e
:
605 if hasattr(e
, 'code') and isinstance(e
.code
, int):
610 print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time
))
613 if __name__
== '__main__':